genkit eval changes

.gitignore

@@ -8,5 +8,6 @@ node_modules/
 
 .bin/
 
+.genkit/
 *.log
 *.txt

.vscode/launch.json

@@ -0,0 +1,16 @@
+{
+  // Use IntelliSense to learn about possible attributes.
+  // Hover to view descriptions of existing attributes.
+  // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+  "version": "0.2.0",
+  "configurations": [
+    {
+      "name": "Launch Package",
+      "type": "go",
+      "request": "launch",
+      "mode": "auto",
+      "program": "${workspaceFolder}/src/cmd/cli",
+      "console": "integratedTerminal"
+    }
+  ]
+}

flake.nix

@@ -34,6 +34,17 @@
               google-cloud-sdk
               vim
             ];
+
+            shellHook = ''
+              # Install genkit-cli locally in the project if not already present
+              if [ ! -f "node_modules/.bin/genkit" ]; then
+                echo "Installing genkit-cli locally..."
+                npm install genkit-cli
+              fi
+
+              # Add local node_modules/.bin to PATH
+              export PATH="$PWD/node_modules/.bin:$PATH"
+            '';
           };
         packages.defang-cli = pkgs.callPackage ./pkgs/defang/cli.nix { };
         packages.defang-bin = pkgs.callPackage ./pkgs/defang { };

pkgs/defang/cli.nix

@@ -7,7 +7,7 @@ buildGo124Module {
   pname = "defang-cli";
   version = "git";
   src = lib.cleanSource ../../src;
-  vendorHash = "sha256-VYJjG99g66S8V6GzqdZngnHESaVkQ4C6Tl2VsFMVCLA="; # TODO: use fetchFromGitHub
+  vendorHash = "sha256-VKC3eh2cx7gTNymUTsaJxLgjpvt+TFhZW4HpRRoZdtg="; # TODO: use fetchFromGitHub
 
   subPackages = [ "cmd/cli" ];
 

src/.gitignore

@@ -11,3 +11,4 @@ distx/
 defang.exe
 samples_examples.json
 knowledge_base.json
+summarizer

src/Makefile

@@ -219,3 +219,144 @@ snapshot:
 	  --snapshot \
 	  --clean \
 	  --skip=publish,announce,sign,notarize,validate
+
+# ==============================================================================
+# Genkit Evaluation System
+# ==============================================================================
+
+# Evaluation directories
+EVAL_DIR := pkg/agent/evaluation
+TESTDATA_DIR := $(EVAL_DIR)/datasets
+GENKIT_CMD_DIR := $(EVAL_DIR)/cmd
+
+# Configurable parameters
+RUNS ?= 10
+EVAL_OUTPUT_FILE_PATTERN := $$(basename $$dataset .json)_results.json
+EVAL_SUMMARY_FILE ?= current_evaluation.json
+
+# Dynamic output directories based on branch and commit
+GIT_BRANCH_RAW := $(shell git rev-parse --abbrev-ref --symbolic-full-name @{u} 2>/dev/null | sed 's|origin/||' || git rev-parse --abbrev-ref HEAD)
+GIT_BRANCH := $(shell echo "$(GIT_BRANCH_RAW)" | sed 's/\//-/g')
+GIT_COMMIT := $(shell git rev-parse --short=6 HEAD)
+EVAL_ROOT_NAME_DIR := $(EVAL_DIR)/testrun-$(GIT_BRANCH)
+EVAL_BASE_DIR := $(EVAL_ROOT_NAME_DIR)-$(GIT_COMMIT)
+RUN_TIMESTAMP := $(shell date +%Y%m%d-%H%M%S)
+EVAL_OUTPUT_DIR := $(EVAL_BASE_DIR)/run-$(RUN_TIMESTAMP)
+
+# Dataset discovery (exclude output files)
+EVAL_DATASETS := $(filter-out $(TESTDATA_DIR)/evaluation_results.json, $(wildcard $(TESTDATA_DIR)/*.json))
+
+# Server management targets
+.PHONY: genkit-start
+genkit-start: ## Start the genkit server for evaluations
+	@echo "Starting Genkit server..."
+	cd $(PWD)/$(GENKIT_CMD_DIR) && genkit start -- go run .
+
+.PHONY: genkit-kill
+genkit-kill: ## Kill any running genkit processes
+	@echo "Killing any existing genkit processes..."
+	@pkill -f "genkit start" || true
+	@pkill -f "go run.*main.go" || true
+	@sleep 2
+	@echo "Genkit processes stopped."
+
+.PHONY: genkit-restart
+genkit-restart: genkit-kill genkit-start ## Kill existing and start fresh genkit server
+
+.PHONY: genkit-check-server
+genkit-check-server: ## Check if genkit server is running
+	@if ! (curl -s http://localhost:4000 > /dev/null 2>&1 || curl -s http://localhost:4033 > /dev/null 2>&1 || curl -s http://localhost:4034 > /dev/null 2>&1); then \
+		echo "Error: Genkit server is not running."; \
+		echo ""; \
+		echo "Please start the server in a separate terminal:"; \
+		echo "  Terminal 1: cd $(PWD) && make genkit-start"; \
+		echo ""; \
+		echo "Then run evaluations from this terminal:"; \
+		echo "  Terminal 2: cd $(PWD) && make genkit-evaluate"; \
+		echo ""; \
+		exit 1; \
+	else \
+		echo " Genkit server detected and running."; \
+	fi
+
+# Evaluation execution targets
+.PHONY: genkit-evaluate
+genkit-evaluate: genkit-check-server ## Run all genkit evaluations (requires genkit server to be running)
+	@echo "Running all evaluations ($(RUNS) runs per dataset)..."
+	@echo "Output directory: $(EVAL_OUTPUT_DIR)"
+	@mkdir -p $(EVAL_OUTPUT_DIR)
+	@for run in $$(seq 1 $(RUNS)); do \
+		echo "Starting run $$run/$(RUNS)..."; \
+		for dataset in $(EVAL_DATASETS); do \
+			echo "  Running evaluation: $$dataset (run $$run)"; \
+			cd $(PWD)/$(GENKIT_CMD_DIR) && genkit eval:flow defang-cli \
+				--input "../datasets/$$(basename $$dataset)" \
+				--output "$(PWD)/$(EVAL_OUTPUT_DIR)/$(EVAL_OUTPUT_FILE_PATTERN)" || true; \
+		done; \
+	done
+	@echo "Generating evaluation summary..."
+	@cd $(PWD)/$(EVAL_DIR)/tools/summarizer && go run . \
+		-path ../../testrun-$(GIT_BRANCH)-$(GIT_COMMIT) \
+		-output ../../$(EVAL_SUMMARY_FILE)
+	@echo "Summary saved to $(EVAL_DIR)/$(EVAL_SUMMARY_FILE)"
+
+.PHONY: genkit-summarize
+genkit-summarize: ## Generate summary of current evaluation results
+	@echo "Generating evaluation summary for $(EVAL_BASE_DIR)..."
+	@cd $(PWD)/$(EVAL_DIR)/tools/summarizer && go run . \
+		-path ../../testrun-$(GIT_BRANCH)-$(GIT_COMMIT) \
+		-output ../../$(EVAL_SUMMARY_FILE)
+	@echo "Summary saved to $(EVAL_DIR)/$(EVAL_SUMMARY_FILE)"
+
+# Utility targets
+.PHONY: genkit-list-datasets
+genkit-list-datasets: ## List all available evaluation datasets
+	@echo "Available evaluation datasets:"
+	@for dataset in $(EVAL_DATASETS); do echo "  $$(basename $$dataset)"; done
+
+.PHONY: genkit-clean
+genkit-clean: ## Clean branch evaluation results
+	@echo "Cleaning branch evaluation results: $(EVAL_BASE_DIR)"
+	@rm -rf $(PWD)/$(GENKIT_CMD_DIR)/.genkit/evals/*.json
+	@rm -rf $(EVAL_ROOT_NAME_DIR)-*
+	@echo "Cleanup completed."
+
+# Help and documentation
+.PHONY: genkit-help
+genkit-help: ## Show genkit evaluation help
+	@echo "=== Genkit Evaluation System ==="
+	@echo ""
+	@echo "Server Management:"
+	@echo "  genkit-start          Start the genkit server (run in separate terminal)"
+	@echo "  genkit-kill           Kill any running genkit processes"
+	@echo "  genkit-restart        Kill existing and start fresh genkit server"
+	@echo ""
+	@echo "Evaluation Execution:"
+	@echo "  genkit-evaluate       Run all evaluations (default: $(RUNS) runs per dataset)"
+	@echo ""
+	@echo "Utilities:"
+	@echo "  genkit-clean          Clean current branch/commit evaluation results"
+	@echo "  genkit-help           Show this help message"
+	@echo ""
+	@echo "Configurable Parameters:"
+	@echo "  RUNS=N                           Number of runs per dataset (default: $(RUNS))"
+	@echo "  EVAL_OUTPUT_FILE_PATTERN=pattern Output filename pattern (default: dataset_results.json)"
+	@echo "  EVAL_SUMMARY_FILE=filename       Summary output filename (default: $(EVAL_SUMMARY_FILE))"
+	@echo ""
+	@echo "Directory Structure:"
+	@echo "  Base directory: $(EVAL_BASE_DIR)"
+	@echo "  Current run:    $(EVAL_OUTPUT_DIR)"
+	@echo ""
+	@echo "Workflow (Two Terminal Setup):"
+	@echo "  Terminal 1: make genkit-start    # Keep running"
+	@echo "  Terminal 2: make genkit-evaluate # Run evaluations"
+	@echo ""
+	@echo "Usage Examples:"
+	@echo "  make genkit-evaluate                                    # Default 10 runs"
+	@echo "  make genkit-evaluate RUNS=5                             # Custom run count"
+	@echo "  make genkit-evaluate EVAL_SUMMARY_FILE=my_results.json  # Custom summary file"
+	@echo ""
+	@echo "Troubleshooting:"
+	@echo "  - If evaluations fail: make genkit-restart (in Terminal 1)"
+	@echo "  - Clean old results:   make genkit-clean"
+	@echo ""

src/cmd/cli/command/commands.go

@@ -15,6 +15,7 @@ import (
 
 	"github.com/AlecAivazis/survey/v2"
 	"github.com/DefangLabs/defang/src/pkg"
+	"github.com/DefangLabs/defang/src/pkg/agent"
 	"github.com/DefangLabs/defang/src/pkg/cli"
 	cliClient "github.com/DefangLabs/defang/src/pkg/cli/client"
 	"github.com/DefangLabs/defang/src/pkg/cli/client/byoc"
@@ -32,6 +33,7 @@ import (
 	"github.com/DefangLabs/defang/src/pkg/setup"
 	"github.com/DefangLabs/defang/src/pkg/surveyor"
 	"github.com/DefangLabs/defang/src/pkg/term"
+	"github.com/DefangLabs/defang/src/pkg/timeutils"
 	"github.com/DefangLabs/defang/src/pkg/track"
 	"github.com/DefangLabs/defang/src/pkg/types"
 	defangv1 "github.com/DefangLabs/defang/src/protos/io/defang/v1"
@@ -388,6 +390,20 @@ var RootCmd = &cobra.Command{
 
 		return err
 	},
+
+	RunE: func(cmd *cobra.Command, args []string) error {
+		if nonInteractive {
+			return nil
+		}
+
+		ctx := cmd.Context()
+		err := login.InteractiveRequireLoginAndToS(ctx, client, getCluster())
+		if err != nil {
+			return err
+		}
+
+		return agent.New(ctx, getCluster(), &providerID, agent.DefaultSystemPrompt).Start()
+	},
 }
 
 var loginCmd = &cobra.Command{
@@ -805,11 +821,11 @@ var debugCmd = &cobra.Command{
 		}
 
 		now := time.Now()
-		sinceTs, err := cli.ParseTimeOrDuration(since, now)
+		sinceTs, err := timeutils.ParseTimeOrDuration(since, now)
 		if err != nil {
 			return fmt.Errorf("invalid 'since' time: %w", err)
 		}
-		untilTs, err := cli.ParseTimeOrDuration(until, now)
+		untilTs, err := timeutils.ParseTimeOrDuration(until, now)
 		if err != nil {
 			return fmt.Errorf("invalid 'until' time: %w", err)
 		}

src/cmd/cli/command/compose.go

@@ -19,6 +19,7 @@ import (
 	"github.com/DefangLabs/defang/src/pkg/logs"
 	"github.com/DefangLabs/defang/src/pkg/modes"
 	"github.com/DefangLabs/defang/src/pkg/term"
+	"github.com/DefangLabs/defang/src/pkg/timeutils"
 	"github.com/DefangLabs/defang/src/pkg/track"
 	"github.com/DefangLabs/defang/src/pkg/types"
 	defangv1 "github.com/DefangLabs/defang/src/protos/io/defang/v1"
@@ -129,7 +130,7 @@ func makeComposeUpCmd() *cobra.Command {
 				term.Warnf("Defang cannot monitor status of the following managed service(s): %v.\n   To check if the managed service is up, check the status of the service which depends on it.", managedServices)
 			}
 
-			deploy, project, err := cli.ComposeUp(ctx, project, client, provider, upload, mode)
+			deploy, project, err := cli.ComposeUp(ctx, client, provider, cli.ComposeUpParams{Project: project, UploadMode: upload, Mode: mode})
 			if err != nil {
 				return handleComposeUpErr(ctx, err, project, provider)
 			}
@@ -452,7 +453,7 @@ func makeComposeConfigCmd() *cobra.Command {
 				return err
 			}
 
-			_, _, err = cli.ComposeUp(ctx, project, client, provider, compose.UploadModeIgnore, modes.ModeUnspecified)
+			_, _, err = cli.ComposeUp(ctx, client, provider, cli.ComposeUpParams{Project: project, UploadMode: compose.UploadModeIgnore, Mode: modes.ModeUnspecified})
 			if !errors.Is(err, dryrun.ErrDryRun) {
 				return err
 			}
@@ -569,12 +570,12 @@ func handleLogsCmd(cmd *cobra.Command, args []string) error {
 	}
 
 	now := time.Now()
-	sinceTs, err := cli.ParseTimeOrDuration(since, now)
+	sinceTs, err := timeutils.ParseTimeOrDuration(since, now)
 	if err != nil {
 		return fmt.Errorf("invalid 'since' duration or time: %w", err)
 	}
 	sinceTs = sinceTs.UTC()
-	untilTs, err := cli.ParseTimeOrDuration(until, now)
+	untilTs, err := timeutils.ParseTimeOrDuration(until, now)
 	if err != nil {
 		return fmt.Errorf("invalid 'until' duration or time: %w", err)
 	}

src/cmd/cli/command/mcp.go

@@ -5,9 +5,9 @@ import (
 	"os"
 	"path/filepath"
 
+	agentTools "github.com/DefangLabs/defang/src/pkg/agent/tools"
 	cliClient "github.com/DefangLabs/defang/src/pkg/cli/client"
 	"github.com/DefangLabs/defang/src/pkg/mcp"
-	"github.com/DefangLabs/defang/src/pkg/mcp/tools"
 	"github.com/DefangLabs/defang/src/pkg/term"
 	"github.com/mark3labs/mcp-go/server"
 	"github.com/spf13/cobra"
@@ -50,7 +50,7 @@ var mcpServerCmd = &cobra.Command{
 
 		// Create a new MCP server
 		term.Debug("Creating MCP server")
-		s, err := mcp.NewDefangMCPServer(RootCmd.Version, cluster, &providerID, mcpClient, tools.DefaultToolCLI{})
+		s, err := mcp.NewDefangMCPServer(RootCmd.Version, cluster, &providerID, mcpClient, agentTools.DefaultToolCLI{})
 		if err != nil {
 			return fmt.Errorf("failed to create MCP server: %w", err)
 		}