From 80386f6113f4f54c7963bebd5d07c8e881ff566b Mon Sep 17 00:00:00 2001 From: Eric Liu Date: Fri, 10 Oct 2025 18:00:32 -0700 Subject: [PATCH 01/18] add last 10 terminal outputs for tracking for errors on command. also add terminal logs to tracking leading up to debug prompt. --- src/cmd/cli/command/commands.go | 11 +++- src/cmd/cli/command/compose.go | 11 ++-- src/pkg/term/colorizer.go | 92 +++++++++++++++++++++++++++++---- src/pkg/term/colorizer_test.go | 47 +++++++++++++++-- src/pkg/track/track.go | 24 +++++++++ 5 files changed, 165 insertions(+), 20 deletions(-) diff --git a/src/cmd/cli/command/commands.go b/src/cmd/cli/command/commands.go index 0285ef7ab..496542620 100644 --- a/src/cmd/cli/command/commands.go +++ b/src/cmd/cli/command/commands.go @@ -350,11 +350,20 @@ var RootCmd = &cobra.Command{ // Use "defer" to track any errors that occur during the command defer func() { var errString = "" + logProps := []track.Property{} if err != nil { errString = err.Error() + + // on error, also log the recent terminal messages + messages := term.DefaultTerm.GetAllMessages() + logProps = append(logProps, track.M("logs", messages)...) } - track.Cmd(cmd, "Invoked", P("args", args), P("err", errString), P("non-interactive", nonInteractive), P("provider", providerID)) + props := []track.Property{ + P("args", args), P("err", errString), P("non-interactive", nonInteractive), P("provider", providerID), + } + props = append(props, logProps...) + track.Cmd(cmd, "Invoked", props...) }() // Do this first, since any errors will be printed to the console diff --git a/src/cmd/cli/command/compose.go b/src/cmd/cli/command/compose.go index 634aa8a66..56a565ac4 100644 --- a/src/cmd/cli/command/compose.go +++ b/src/cmd/cli/command/compose.go @@ -91,7 +91,7 @@ func makeComposeUpCmd() *cobra.Command { return err } - track.Evt("Debug Prompted", P("loadErr", loadErr)) + track.EvtWithTerm("Debug Prompted", P("loadErr", loadErr)) return cli.InteractiveDebugForClientError(ctx, client, project, loadErr) } @@ -245,7 +245,7 @@ func handleComposeUpErr(ctx context.Context, err error, project *compose.Project } term.Error("Error:", cliClient.PrettyError(err)) - track.Evt("Debug Prompted", P("composeErr", err)) + track.EvtWithTerm("Debug Prompted", P("composeErr", err)) return cli.InteractiveDebugForClientError(ctx, client, project, err) } @@ -260,7 +260,10 @@ func handleTailAndMonitorErr(ctx context.Context, err error, client *cliClient.G if nonInteractive { printDefangHint("To debug the deployment, do:", debugConfig.String()) } else { - track.Evt("Debug Prompted", P("failedServices", debugConfig.FailedServices), P("etag", debugConfig.Deployment), P("reason", errDeploymentFailed)) + track.EvtWithTerm("Debug Prompted", P("failedServices", debugConfig.FailedServices), + P("etag", debugConfig.Deployment), + P("reason", errDeploymentFailed), + ) // Call the AI debug endpoint using the original command context (not the tail ctx which is canceled) if nil != cli.InteractiveDebugDeployment(ctx, client, debugConfig) { @@ -456,7 +459,7 @@ func makeComposeConfigCmd() *cobra.Command { return err } - track.Evt("Debug Prompted", P("loadErr", loadErr)) + track.EvtWithTerm("Debug Prompted", P("loadErr", loadErr)) return cli.InteractiveDebugForClientError(ctx, client, project, loadErr) } diff --git a/src/pkg/term/colorizer.go b/src/pkg/term/colorizer.go index fab546074..1459adbd7 100644 --- a/src/pkg/term/colorizer.go +++ b/src/pkg/term/colorizer.go @@ -12,6 +12,54 @@ import ( "golang.org/x/term" ) +// DefaultBufferSize is the default number of recent messages to keep in the circular buffer +var DefaultBufferSize = 10 + +type circularBuffer struct { + size int + linesWritten int + index int + data []string +} + +func (c *circularBuffer) write(msg string) { + c.linesWritten++ + if len(c.data) < c.size { + c.data = append(c.data, msg) + } else { + c.data[c.index] = msg + } + c.index = (c.index + 1) % c.size +} + +func (c *circularBuffer) read() []string { + if c.linesWritten <= c.size { + return slices.Clone(c.data) + } + + messages := make([]string, 0, c.size) + startIdx := c.index + + // Collect messages in chronological order + for i := range c.size { + idx := (startIdx + i) % c.size + messages = append(messages, c.data[idx]) + } + return messages +} + +func NewCircularBuffer(bufferSize int) circularBuffer { + if bufferSize <= 0 { + bufferSize = 1 // ensure at least 1 element + } + return circularBuffer{ + size: bufferSize, + linesWritten: 0, + index: 0, + data: make([]string, 0, bufferSize), + } +} + type Term struct { stdin FileReader stdout, stderr io.Writer @@ -22,6 +70,7 @@ type Term struct { hasDarkBg bool warnings []string + buffer circularBuffer } var DefaultTerm = NewTerm(os.Stdin, os.Stdout, os.Stderr) @@ -56,6 +105,7 @@ func NewTerm(stdin FileReader, stdout, stderr io.Writer) *Term { stderr: stderr, out: termenv.NewOutput(stdout), err: termenv.NewOutput(stderr), + buffer: NewCircularBuffer(DefaultBufferSize), } t.hasDarkBg = t.out.HasDarkBackground() if hasTermInEnv() { @@ -70,6 +120,11 @@ func (t Term) Stdio() (FileReader, termenv.File, io.Writer) { return t.stdin, t.out.TTY(), t.err } +// GetAllMessages returns all messages currently stored in the buffer in chronological order +func (t Term) GetAllMessages() []string { + return t.buffer.read() +} + func (t *Term) ForceColor(color bool) { if color { t.out = termenv.NewOutput(t.stdout, termenv.WithProfile(termenv.ANSI)) @@ -178,62 +233,79 @@ func ensurePrefix(s string, prefix string) string { return prefix + s } +func (t *Term) output(c Color, v ...any) (int, error) { + msg := fmt.Sprint(v...) + t.buffer.write(msg) + return output(t.out, c, msg) +} + func (t *Term) Printc(c Color, v ...any) (int, error) { - return output(t.out, c, fmt.Sprint(v...)) + return t.output(c, v...) } func (t *Term) Print(v ...any) (int, error) { + t.buffer.write(fmt.Sprint(v...)) return fmt.Fprint(t.out, v...) } func (t *Term) Println(v ...any) (int, error) { - return fmt.Fprint(t.out, ensureNewline(fmt.Sprintln(v...))) + text := ensureNewline(fmt.Sprintln(v...)) + t.buffer.write(text) + return fmt.Fprint(t.out, text) } func (t *Term) Printf(format string, v ...any) (int, error) { - return fmt.Fprint(t.out, ensureNewline(fmt.Sprintf(format, v...))) + text := ensureNewline(fmt.Sprintf(format, v...)) + t.buffer.write(text) + return fmt.Fprint(t.out, text) } func (t *Term) Debug(v ...any) (int, error) { if !t.debug { return 0, nil } - return output(t.out, DebugColor, ensurePrefix(fmt.Sprintln(v...), " - ")) + return t.output(DebugColor, ensurePrefix(fmt.Sprintln(v...), " - ")) } func (t *Term) Debugf(format string, v ...any) (int, error) { if !t.debug { return 0, nil } - return output(t.out, DebugColor, ensureNewline(ensurePrefix(fmt.Sprintf(format, v...), " - "))) + s := ensureNewline(ensurePrefix(fmt.Sprintf(format, v...), " - ")) + return t.output(DebugColor, s) } func (t *Term) Info(v ...any) (int, error) { - return output(t.out, InfoColor, ensurePrefix(fmt.Sprintln(v...), " * ")) + s := ensurePrefix(fmt.Sprintln(v...), " * ") + return t.output(InfoColor, s) } func (t *Term) Infof(format string, v ...any) (int, error) { - return output(t.out, InfoColor, ensureNewline(ensurePrefix(fmt.Sprintf(format, v...), " * "))) + s := ensureNewline(ensurePrefix(fmt.Sprintf(format, v...), " * ")) + return t.output(InfoColor, s) } func (t *Term) Warn(v ...any) (int, error) { msg := ensurePrefix(fmt.Sprintln(v...), " ! ") t.warnings = append(t.warnings, msg) - return output(t.out, WarnColor, msg) + return t.output(WarnColor, msg) } func (t *Term) Warnf(format string, v ...any) (int, error) { msg := ensureNewline(ensurePrefix(fmt.Sprintf(format, v...), " ! ")) t.warnings = append(t.warnings, msg) - return output(t.out, WarnColor, msg) + return t.output(WarnColor, msg) } func (t *Term) Error(v ...any) (int, error) { - return output(t.err, ErrorColor, fmt.Sprintln(v...)) + msg := fmt.Sprintln(v...) + t.buffer.write(msg) + return output(t.err, ErrorColor, msg) } func (t *Term) Errorf(format string, v ...any) (int, error) { line := ensureNewline(fmt.Sprintf(format, v...)) + t.buffer.write(line) return output(t.err, ErrorColor, line) } diff --git a/src/pkg/term/colorizer_test.go b/src/pkg/term/colorizer_test.go index 2ce889179..880668b4d 100644 --- a/src/pkg/term/colorizer_test.go +++ b/src/pkg/term/colorizer_test.go @@ -9,6 +9,7 @@ import ( "testing" "github.com/muesli/termenv" + "github.com/stretchr/testify/assert" ) func TestOutput(t *testing.T) { @@ -218,13 +219,13 @@ func TestFlushWarnings(t *testing.T) { for i, test := range tests { t.Run(strconv.Itoa(i), func(t *testing.T) { var stdout, stderr bytes.Buffer - term := NewTerm(os.Stdin, &stdout, &stderr) + termWriter := NewTerm(os.Stdin, &stdout, &stderr) for _, warning := range test.warnings { - term.Warn(warning) + termWriter.Warn(warning) } - bytesWritten, err := term.FlushWarnings() + bytesWritten, err := termWriter.FlushWarnings() if (err != nil) != test.expectErr { t.Errorf("FlushWarnings() error = %v, expectErr %v", err, test.expectErr) } @@ -237,9 +238,45 @@ func TestFlushWarnings(t *testing.T) { t.Errorf("FlushWarnings() expected %d byteWritten, got %d", bytesInExpected, bytesWritten) } - if term.getAllWarnings() != nil { - t.Errorf("after FlushWarnings() expected no warnings, got %v", term.getAllWarnings()) + if termWriter.getAllWarnings() != nil { + t.Errorf("after FlushWarnings() expected no warnings, got %v", termWriter.getAllWarnings()) } }) } } + +func TestWriteToBuffer(t *testing.T) { + var stdout, stderr bytes.Buffer + + originalBufferSize := DefaultBufferSize + t.Cleanup(func() { + DefaultBufferSize = originalBufferSize + }) + DefaultBufferSize = 5 + + termWriter := NewTerm(os.Stdin, &stdout, &stderr) + + // no messages initially + currentMessages := termWriter.GetAllMessages() + assert.Empty(t, currentMessages) + + // add messages, less than buffer capacity + termWriter.Info("message 1") + termWriter.Info("message 2") + termWriter.Info("message 3") + termWriter.Info("message 4") + + // sanity check + currentMessages = termWriter.GetAllMessages() + expectedMessages := []string{" * message 1\n", " * message 2\n", " * message 3\n", " * message 4\n"} + assert.Equal(t, expectedMessages, currentMessages) + + // add messages to be over buffer capacity + termWriter.Info("message A") + termWriter.Info("message B") + + // check only the last number of message are kept + currentMessages = termWriter.GetAllMessages() + expectedMessages = []string{" * message 2\n", " * message 3\n", " * message 4\n", " * message A\n", " * message B\n"} + assert.Equal(t, expectedMessages, currentMessages) +} diff --git a/src/pkg/track/track.go b/src/pkg/track/track.go index ea60eac79..b1d3cecd2 100644 --- a/src/pkg/track/track.go +++ b/src/pkg/track/track.go @@ -1,6 +1,7 @@ package track import ( + "fmt" "strings" "sync" @@ -56,6 +57,27 @@ func FlushAllTracking() { trackWG.Wait() } +// function to break a set of messages into smaller chunks for tracking +// There is a set size limit per property for tracking +var M = func(name string, message []string) []Property { + const maxMessagePerProperty = 3 + + var trackMsg []Property + for i := 0; i < len(message); i += maxMessagePerProperty { + end := min(i+maxMessagePerProperty, len(message)) + propName := fmt.Sprintf("%s-%d", name, i/maxMessagePerProperty+1) + trackMsg = append(trackMsg, P(propName, message[i:end])) + } + return trackMsg +} + +func EvtWithTerm(eventName string, extraProps ...Property) { + messages := term.DefaultTerm.GetAllMessages() + logProps := M("logs", messages) + allProps := append(extraProps, logProps...) + Evt(eventName, allProps...) +} + func isCompletionCommand(cmd *cobra.Command) bool { return cmd.Name() == cobra.ShellCompRequestCmd || (cmd.Parent() != nil && cmd.Parent().Name() == "completion") } @@ -76,10 +98,12 @@ func Cmd(cmd *cobra.Command, verb string, props ...Property) { command = c.Name() + "-" + command } }) + props = append(props, P("CalledAs", calledAs), P("version", cmd.Root().Version), ) + cmd.Flags().Visit(func(f *pflag.Flag) { props = append(props, P(f.Name, f.Value)) }) From cc28f37499fb4b56f1a0fc49f37ddeb576ccc885 Mon Sep 17 00:00:00 2001 From: Eric Liu Date: Sat, 11 Oct 2025 22:27:17 -0700 Subject: [PATCH 02/18] clean up --- src/cmd/cli/command/commands.go | 2 +- src/pkg/track/track.go | 10 ++-- src/pkg/track/track_test.go | 96 +++++++++++++++++++++++++++++++++ 3 files changed, 103 insertions(+), 5 deletions(-) create mode 100644 src/pkg/track/track_test.go diff --git a/src/cmd/cli/command/commands.go b/src/cmd/cli/command/commands.go index 496542620..875a3cd20 100644 --- a/src/cmd/cli/command/commands.go +++ b/src/cmd/cli/command/commands.go @@ -356,7 +356,7 @@ var RootCmd = &cobra.Command{ // on error, also log the recent terminal messages messages := term.DefaultTerm.GetAllMessages() - logProps = append(logProps, track.M("logs", messages)...) + logProps = append(logProps, track.ChunkMessages("logs", messages)...) } props := []track.Property{ diff --git a/src/pkg/track/track.go b/src/pkg/track/track.go index b1d3cecd2..2076e6255 100644 --- a/src/pkg/track/track.go +++ b/src/pkg/track/track.go @@ -12,6 +12,10 @@ import ( "github.com/spf13/pflag" ) +// maxMessagePerProperty is the maximum number of messages to include in a single tracking property. +// 3 seems to be a reasonable number to avoid exceeding size limits. +const maxMessagePerProperty = 3 + var disableAnalytics = pkg.GetenvBool("DEFANG_DISABLE_ANALYTICS") type Property = cliClient.Property @@ -59,9 +63,7 @@ func FlushAllTracking() { // function to break a set of messages into smaller chunks for tracking // There is a set size limit per property for tracking -var M = func(name string, message []string) []Property { - const maxMessagePerProperty = 3 - +func ChunkMessages(name string, message []string) []Property { var trackMsg []Property for i := 0; i < len(message); i += maxMessagePerProperty { end := min(i+maxMessagePerProperty, len(message)) @@ -73,7 +75,7 @@ var M = func(name string, message []string) []Property { func EvtWithTerm(eventName string, extraProps ...Property) { messages := term.DefaultTerm.GetAllMessages() - logProps := M("logs", messages) + logProps := ChunkMessages("logs", messages) allProps := append(extraProps, logProps...) Evt(eventName, allProps...) } diff --git a/src/pkg/track/track_test.go b/src/pkg/track/track_test.go new file mode 100644 index 000000000..569ce7694 --- /dev/null +++ b/src/pkg/track/track_test.go @@ -0,0 +1,96 @@ +package track + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestChunkMessages(t *testing.T) { + tests := []struct { + name string + prefix string + messages []string + want int // number of expected chunks + }{ + { + name: "empty messages", + prefix: "logs", + messages: []string{}, + want: 0, + }, + { + name: "single message", + prefix: "logs", + messages: []string{"message1"}, + want: 1, + }, + { + name: "three messages - one chunk", + prefix: "logs", + messages: []string{"msg1", "msg2", "msg3"}, + want: 1, + }, + { + name: "four messages - two chunks", + prefix: "logs", + messages: []string{"msg1", "msg2", "msg3", "msg4"}, + want: 2, + }, + { + name: "six messages - two full chunks", + prefix: "logs", + messages: []string{"msg1", "msg2", "msg3", "msg4", "msg5", "msg6"}, + want: 2, + }, + { + name: "seven messages - three chunks", + prefix: "debug", + messages: []string{"msg1", "msg2", "msg3", "msg4", "msg5", "msg6", "msg7"}, + want: 3, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := ChunkMessages(tt.prefix, tt.messages) + + // Check number of chunks + assert.Equal(t, tt.want, len(result), "incorrect number of chunks") + + if len(tt.messages) == 0 { + return // No more checks needed for empty input + } + + // Verify chunk sizes and property names + totalMessages := 0 + for i, prop := range result { + // Check property name format + expectedName := fmt.Sprintf("%s-%d", tt.prefix, i+1) + assert.Equal(t, expectedName, prop.Name, "incorrect property name") + + // Check that value is []string + msgs, ok := prop.Value.([]string) + assert.True(t, ok, "property value should be []string") + + // Check chunk size + if i < len(result)-1 { + // All chunks except the last should have maxMessagePerProperty messages + assert.Equal(t, maxMessagePerProperty, len(msgs), "non-final chunk has incorrect size") + } else { + // Last chunk should have remaining messages (1-3 messages) + expectedLastSize := len(tt.messages) - (i * maxMessagePerProperty) + assert.Equal(t, expectedLastSize, len(msgs), "final chunk has incorrect size") + assert.LessOrEqual(t, len(msgs), maxMessagePerProperty, "final chunk exceeds max size") + assert.Greater(t, len(msgs), 0, "final chunk should not be empty") + } + + totalMessages += len(msgs) + } + + // Verify all messages are preserved + assert.Equal(t, len(tt.messages), totalMessages, "total message count should be preserved") + }) + } +} From 5e67c57dec1d8d17530b3cc7c2d5253d1d10dbf9 Mon Sep 17 00:00:00 2001 From: Eric Liu Date: Fri, 17 Oct 2025 15:33:12 -0700 Subject: [PATCH 03/18] review update. make circular buffer it's own package and tests. Make circular buffer a generic --- src/pkg/datastructs/circularbuffer.go | 48 +++++++++++++ src/pkg/datastructs/circularbuffer_test.go | 23 ++++++ src/pkg/term/colorizer.go | 67 +++--------------- src/pkg/term/colorizer_test.go | 7 +- src/pkg/track/track.go | 22 +++--- src/pkg/track/track_test.go | 82 +++++++++------------- 6 files changed, 133 insertions(+), 116 deletions(-) create mode 100644 src/pkg/datastructs/circularbuffer.go create mode 100644 src/pkg/datastructs/circularbuffer_test.go diff --git a/src/pkg/datastructs/circularbuffer.go b/src/pkg/datastructs/circularbuffer.go new file mode 100644 index 000000000..bfcc928a9 --- /dev/null +++ b/src/pkg/datastructs/circularbuffer.go @@ -0,0 +1,48 @@ +package datastructs + +var DefaultBufferSize = 10 + +type CircularBuffer[T any] struct { + size int + entries int + index int + data []T +} + +func (c *CircularBuffer[T]) Add(item T) { + c.entries++ + c.data[c.index] = item + c.index = (c.index + 1) % c.size +} + +func (c *CircularBuffer[T]) Get() []T { + maxItems := min(c.entries, c.size) + items := make([]T, 0, maxItems) + startIdx := c.index + + // the c.index points to the next write position (ie. oldest entry) if the buffer is full, + // otherwise if the buffer is not full then c.index does not point to the older entry so we + // need to start from index 0 + if c.entries < c.size { + startIdx = 0 + } + + // Collect items in chronological order + for i := range maxItems { + idx := (startIdx + i) % c.size + items = append(items, c.data[idx]) + } + return items +} + +func NewCircularBuffer[T any](bufferSize int) CircularBuffer[T] { + if bufferSize <= 0 { + bufferSize = 1 // ensure at least 1 element + } + return CircularBuffer[T]{ + size: bufferSize, + entries: 0, + index: 0, + data: make([]T, bufferSize), + } +} diff --git a/src/pkg/datastructs/circularbuffer_test.go b/src/pkg/datastructs/circularbuffer_test.go new file mode 100644 index 000000000..e2281d5a7 --- /dev/null +++ b/src/pkg/datastructs/circularbuffer_test.go @@ -0,0 +1,23 @@ +package datastructs + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestCircularBuffer(t *testing.T) { + buffer := NewCircularBuffer[int](3) + + assert.Equal(t, []int{}, buffer.Get()) + buffer.Add(1) + assert.Equal(t, []int{1}, buffer.Get()) + buffer.Add(2) + assert.Equal(t, []int{1, 2}, buffer.Get()) + buffer.Add(3) + assert.Equal(t, []int{1, 2, 3}, buffer.Get()) + buffer.Add(4) + assert.Equal(t, []int{2, 3, 4}, buffer.Get()) + buffer.Add(5) + assert.Equal(t, []int{3, 4, 5}, buffer.Get()) +} diff --git a/src/pkg/term/colorizer.go b/src/pkg/term/colorizer.go index 1459adbd7..7cbae12e3 100644 --- a/src/pkg/term/colorizer.go +++ b/src/pkg/term/colorizer.go @@ -8,58 +8,11 @@ import ( "slices" "strings" + "github.com/DefangLabs/defang/src/pkg/datastructs" "github.com/muesli/termenv" "golang.org/x/term" ) -// DefaultBufferSize is the default number of recent messages to keep in the circular buffer -var DefaultBufferSize = 10 - -type circularBuffer struct { - size int - linesWritten int - index int - data []string -} - -func (c *circularBuffer) write(msg string) { - c.linesWritten++ - if len(c.data) < c.size { - c.data = append(c.data, msg) - } else { - c.data[c.index] = msg - } - c.index = (c.index + 1) % c.size -} - -func (c *circularBuffer) read() []string { - if c.linesWritten <= c.size { - return slices.Clone(c.data) - } - - messages := make([]string, 0, c.size) - startIdx := c.index - - // Collect messages in chronological order - for i := range c.size { - idx := (startIdx + i) % c.size - messages = append(messages, c.data[idx]) - } - return messages -} - -func NewCircularBuffer(bufferSize int) circularBuffer { - if bufferSize <= 0 { - bufferSize = 1 // ensure at least 1 element - } - return circularBuffer{ - size: bufferSize, - linesWritten: 0, - index: 0, - data: make([]string, 0, bufferSize), - } -} - type Term struct { stdin FileReader stdout, stderr io.Writer @@ -70,7 +23,7 @@ type Term struct { hasDarkBg bool warnings []string - buffer circularBuffer + buffer datastructs.CircularBuffer[string] } var DefaultTerm = NewTerm(os.Stdin, os.Stdout, os.Stderr) @@ -105,7 +58,7 @@ func NewTerm(stdin FileReader, stdout, stderr io.Writer) *Term { stderr: stderr, out: termenv.NewOutput(stdout), err: termenv.NewOutput(stderr), - buffer: NewCircularBuffer(DefaultBufferSize), + buffer: datastructs.NewCircularBuffer[string](datastructs.DefaultBufferSize), } t.hasDarkBg = t.out.HasDarkBackground() if hasTermInEnv() { @@ -122,7 +75,7 @@ func (t Term) Stdio() (FileReader, termenv.File, io.Writer) { // GetAllMessages returns all messages currently stored in the buffer in chronological order func (t Term) GetAllMessages() []string { - return t.buffer.read() + return t.buffer.Get() } func (t *Term) ForceColor(color bool) { @@ -235,7 +188,7 @@ func ensurePrefix(s string, prefix string) string { func (t *Term) output(c Color, v ...any) (int, error) { msg := fmt.Sprint(v...) - t.buffer.write(msg) + t.buffer.Add(msg) return output(t.out, c, msg) } @@ -244,19 +197,19 @@ func (t *Term) Printc(c Color, v ...any) (int, error) { } func (t *Term) Print(v ...any) (int, error) { - t.buffer.write(fmt.Sprint(v...)) + t.buffer.Add(fmt.Sprint(v...)) return fmt.Fprint(t.out, v...) } func (t *Term) Println(v ...any) (int, error) { text := ensureNewline(fmt.Sprintln(v...)) - t.buffer.write(text) + t.buffer.Add(text) return fmt.Fprint(t.out, text) } func (t *Term) Printf(format string, v ...any) (int, error) { text := ensureNewline(fmt.Sprintf(format, v...)) - t.buffer.write(text) + t.buffer.Add(text) return fmt.Fprint(t.out, text) } @@ -299,13 +252,13 @@ func (t *Term) Warnf(format string, v ...any) (int, error) { func (t *Term) Error(v ...any) (int, error) { msg := fmt.Sprintln(v...) - t.buffer.write(msg) + t.buffer.Add(msg) return output(t.err, ErrorColor, msg) } func (t *Term) Errorf(format string, v ...any) (int, error) { line := ensureNewline(fmt.Sprintf(format, v...)) - t.buffer.write(line) + t.buffer.Add(line) return output(t.err, ErrorColor, line) } diff --git a/src/pkg/term/colorizer_test.go b/src/pkg/term/colorizer_test.go index 880668b4d..226861650 100644 --- a/src/pkg/term/colorizer_test.go +++ b/src/pkg/term/colorizer_test.go @@ -8,6 +8,7 @@ import ( "strings" "testing" + "github.com/DefangLabs/defang/src/pkg/datastructs" "github.com/muesli/termenv" "github.com/stretchr/testify/assert" ) @@ -248,11 +249,11 @@ func TestFlushWarnings(t *testing.T) { func TestWriteToBuffer(t *testing.T) { var stdout, stderr bytes.Buffer - originalBufferSize := DefaultBufferSize + originalBufferSize := datastructs.DefaultBufferSize t.Cleanup(func() { - DefaultBufferSize = originalBufferSize + datastructs.DefaultBufferSize = originalBufferSize }) - DefaultBufferSize = 5 + datastructs.DefaultBufferSize = 5 termWriter := NewTerm(os.Stdin, &stdout, &stderr) diff --git a/src/pkg/track/track.go b/src/pkg/track/track.go index 2076e6255..7fe9b9b53 100644 --- a/src/pkg/track/track.go +++ b/src/pkg/track/track.go @@ -12,9 +12,7 @@ import ( "github.com/spf13/pflag" ) -// maxMessagePerProperty is the maximum number of messages to include in a single tracking property. -// 3 seems to be a reasonable number to avoid exceeding size limits. -const maxMessagePerProperty = 3 +const chunkSizeInCharacters = 80 // chars per property in tracking event var disableAnalytics = pkg.GetenvBool("DEFANG_DISABLE_ANALYTICS") @@ -61,14 +59,22 @@ func FlushAllTracking() { trackWG.Wait() } +func ChunkMessages(name string, message []string) []Property { + return ChunkMessagesWithSize(name, message, chunkSizeInCharacters) +} + // function to break a set of messages into smaller chunks for tracking // There is a set size limit per property for tracking -func ChunkMessages(name string, message []string) []Property { +func ChunkMessagesWithSize(name string, message []string, maxChunkSize int) []Property { var trackMsg []Property - for i := 0; i < len(message); i += maxMessagePerProperty { - end := min(i+maxMessagePerProperty, len(message)) - propName := fmt.Sprintf("%s-%d", name, i/maxMessagePerProperty+1) - trackMsg = append(trackMsg, P(propName, message[i:end])) + // make the message one long string + messageStr := strings.Join(message, "\n") + + // split the message into chunks of maxChunkSize + for i := 0; i < len(messageStr); i += maxChunkSize { + end := min(i+maxChunkSize, len(messageStr)) + propName := fmt.Sprintf("%s-%d", name, i/maxChunkSize+1) + trackMsg = append(trackMsg, P(propName, messageStr[i:end])) } return trackMsg } diff --git a/src/pkg/track/track_test.go b/src/pkg/track/track_test.go index 569ce7694..b1ff356f7 100644 --- a/src/pkg/track/track_test.go +++ b/src/pkg/track/track_test.go @@ -2,95 +2,81 @@ package track import ( "fmt" + "strings" "testing" "github.com/stretchr/testify/assert" ) func TestChunkMessages(t *testing.T) { + maxChunkSize := 5 tests := []struct { - name string - prefix string - messages []string - want int // number of expected chunks + name string + prefix string + messages []string + expectedChunkContents []string }{ { - name: "empty messages", - prefix: "logs", - messages: []string{}, - want: 0, + name: "empty messages", + prefix: "logs", + messages: []string{}, + expectedChunkContents: []string{}, }, { - name: "single message", - prefix: "logs", - messages: []string{"message1"}, - want: 1, + name: "single message", + prefix: "logs", + messages: []string{"msg"}, + expectedChunkContents: []string{"msg"}, }, { - name: "three messages - one chunk", - prefix: "logs", - messages: []string{"msg1", "msg2", "msg3"}, - want: 1, + name: "three messages - one chunk", + prefix: "logs", + messages: []string{"1", "2", "3"}, + expectedChunkContents: []string{"1\n2\n3"}, }, { - name: "four messages - two chunks", - prefix: "logs", - messages: []string{"msg1", "msg2", "msg3", "msg4"}, - want: 2, - }, - { - name: "six messages - two full chunks", - prefix: "logs", - messages: []string{"msg1", "msg2", "msg3", "msg4", "msg5", "msg6"}, - want: 2, - }, - { - name: "seven messages - three chunks", - prefix: "debug", - messages: []string{"msg1", "msg2", "msg3", "msg4", "msg5", "msg6", "msg7"}, - want: 3, + name: "four messages - 4 chunks", + prefix: "logs", + messages: []string{"123", "456", "789", "abc"}, + expectedChunkContents: []string{"123\n4", "56\n78", "9\nabc"}, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - result := ChunkMessages(tt.prefix, tt.messages) + result := ChunkMessagesWithSize(tt.prefix, tt.messages, maxChunkSize) // Check number of chunks - assert.Equal(t, tt.want, len(result), "incorrect number of chunks") + assert.Equal(t, len(tt.expectedChunkContents), len(result), "incorrect number of chunks") if len(tt.messages) == 0 { return // No more checks needed for empty input } // Verify chunk sizes and property names - totalMessages := 0 for i, prop := range result { // Check property name format expectedName := fmt.Sprintf("%s-%d", tt.prefix, i+1) assert.Equal(t, expectedName, prop.Name, "incorrect property name") - // Check that value is []string - msgs, ok := prop.Value.([]string) - assert.True(t, ok, "property value should be []string") + // Check that value is string + msgs, ok := prop.Value.(string) + assert.True(t, ok, "property value should be string") // Check chunk size if i < len(result)-1 { // All chunks except the last should have maxMessagePerProperty messages - assert.Equal(t, maxMessagePerProperty, len(msgs), "non-final chunk has incorrect size") + assert.Equal(t, maxChunkSize, len(msgs), "non-final chunk has incorrect size") } else { - // Last chunk should have remaining messages (1-3 messages) - expectedLastSize := len(tt.messages) - (i * maxMessagePerProperty) - assert.Equal(t, expectedLastSize, len(msgs), "final chunk has incorrect size") - assert.LessOrEqual(t, len(msgs), maxMessagePerProperty, "final chunk exceeds max size") + // final chunk + assert.LessOrEqual(t, len(msgs), maxChunkSize, "final chunk exceeds max size") assert.Greater(t, len(msgs), 0, "final chunk should not be empty") - } - totalMessages += len(msgs) + lastValue, ok := result[len(result)-1].Value.(string) + assert.True(t, ok, "last chunk value should be string") + assert.True(t, strings.HasSuffix(msgs, lastValue), "final chunk content mismatch") + } } - - // Verify all messages are preserved - assert.Equal(t, len(tt.messages), totalMessages, "total message count should be preserved") }) } } From d799e2e6d43b54d00190fe1cee52687d972d65c5 Mon Sep 17 00:00:00 2001 From: Eric Liu Date: Fri, 17 Oct 2025 15:37:48 -0700 Subject: [PATCH 04/18] update max chunk size comment --- src/pkg/track/track.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pkg/track/track.go b/src/pkg/track/track.go index 7fe9b9b53..330e9db50 100644 --- a/src/pkg/track/track.go +++ b/src/pkg/track/track.go @@ -12,7 +12,7 @@ import ( "github.com/spf13/pflag" ) -const chunkSizeInCharacters = 80 // chars per property in tracking event +const chunkSizeInCharacters = 80 // chars per property in tracking event. This is a conservative guess on the max size var disableAnalytics = pkg.GetenvBool("DEFANG_DISABLE_ANALYTICS") From db7ff38d265c4876dee7cfd3237fa9f243f5db80 Mon Sep 17 00:00:00 2001 From: Eric Liu Date: Mon, 20 Oct 2025 12:53:48 -0700 Subject: [PATCH 05/18] Update src/pkg/datastructs/circularbuffer.go Co-authored-by: Jordan Stephens --- src/pkg/datastructs/circularbuffer.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pkg/datastructs/circularbuffer.go b/src/pkg/datastructs/circularbuffer.go index bfcc928a9..62c1ed660 100644 --- a/src/pkg/datastructs/circularbuffer.go +++ b/src/pkg/datastructs/circularbuffer.go @@ -37,7 +37,7 @@ func (c *CircularBuffer[T]) Get() []T { func NewCircularBuffer[T any](bufferSize int) CircularBuffer[T] { if bufferSize <= 0 { - bufferSize = 1 // ensure at least 1 element + panic("failed to created a circular buffer: cannot have zero elements") } return CircularBuffer[T]{ size: bufferSize, From 196a8cc33f46036b01cf03283dc8d66721a5f5ac Mon Sep 17 00:00:00 2001 From: Eric Liu Date: Mon, 20 Oct 2025 15:03:17 -0700 Subject: [PATCH 06/18] review up - make logs per message rather than chunking --- src/cmd/cli/command/commands.go | 2 +- src/pkg/datastructs/circularbuffer.go | 2 +- src/pkg/track/track.go | 38 ++++++++++++---------- src/pkg/track/track_test.go | 45 ++++++++++----------------- 4 files changed, 41 insertions(+), 46 deletions(-) diff --git a/src/cmd/cli/command/commands.go b/src/cmd/cli/command/commands.go index 875a3cd20..b1b85782a 100644 --- a/src/cmd/cli/command/commands.go +++ b/src/cmd/cli/command/commands.go @@ -356,7 +356,7 @@ var RootCmd = &cobra.Command{ // on error, also log the recent terminal messages messages := term.DefaultTerm.GetAllMessages() - logProps = append(logProps, track.ChunkMessages("logs", messages)...) + logProps = append(logProps, track.MakeEventLogProperties("log", messages)...) } props := []track.Property{ diff --git a/src/pkg/datastructs/circularbuffer.go b/src/pkg/datastructs/circularbuffer.go index 62c1ed660..2c8f8368f 100644 --- a/src/pkg/datastructs/circularbuffer.go +++ b/src/pkg/datastructs/circularbuffer.go @@ -1,6 +1,6 @@ package datastructs -var DefaultBufferSize = 10 +var DefaultBufferSize = 30 type CircularBuffer[T any] struct { size int diff --git a/src/pkg/track/track.go b/src/pkg/track/track.go index 330e9db50..51c3b2053 100644 --- a/src/pkg/track/track.go +++ b/src/pkg/track/track.go @@ -12,9 +12,10 @@ import ( "github.com/spf13/pflag" ) -const chunkSizeInCharacters = 80 // chars per property in tracking event. This is a conservative guess on the max size +const maxPropertyCharacterLength = 255 // chars per property in tracking event var disableAnalytics = pkg.GetenvBool("DEFANG_DISABLE_ANALYTICS") +var logPropertyNamePrefix = "logs" type Property = cliClient.Property @@ -46,7 +47,17 @@ func Evt(name string, props ...Property) { term.Debugf("untracked event %q: %v", name, props) return } - term.Debugf("tracking event %q: %v", name, props) + + // filter out props with a name prefix of logPropertyNamePrefix, they should already be in the debug output + var filteredProps []Property + for _, p := range props { + if strings.HasPrefix(p.Name, logPropertyNamePrefix) { + continue + } + filteredProps = append(filteredProps, p) + } + + term.Debugf("tracking event %q: %v", name, filteredProps) trackWG.Add(1) go func() { defer trackWG.Done() @@ -59,29 +70,24 @@ func FlushAllTracking() { trackWG.Wait() } -func ChunkMessages(name string, message []string) []Property { - return ChunkMessagesWithSize(name, message, chunkSizeInCharacters) -} - // function to break a set of messages into smaller chunks for tracking // There is a set size limit per property for tracking -func ChunkMessagesWithSize(name string, message []string, maxChunkSize int) []Property { +func MakeEventLogProperties(name string, message []string) []Property { var trackMsg []Property - // make the message one long string - messageStr := strings.Join(message, "\n") - - // split the message into chunks of maxChunkSize - for i := 0; i < len(messageStr); i += maxChunkSize { - end := min(i+maxChunkSize, len(messageStr)) - propName := fmt.Sprintf("%s-%d", name, i/maxChunkSize+1) - trackMsg = append(trackMsg, P(propName, messageStr[i:end])) + + for i, msg := range message { + if len(msg) > maxPropertyCharacterLength { + msg = msg[:maxPropertyCharacterLength] + } + propName := fmt.Sprintf("%s-%d", name, i+1) + trackMsg = append(trackMsg, P(propName, msg)) } return trackMsg } func EvtWithTerm(eventName string, extraProps ...Property) { messages := term.DefaultTerm.GetAllMessages() - logProps := ChunkMessages("logs", messages) + logProps := MakeEventLogProperties(logPropertyNamePrefix, messages) allProps := append(extraProps, logProps...) Evt(eventName, allProps...) } diff --git a/src/pkg/track/track_test.go b/src/pkg/track/track_test.go index b1ff356f7..808143c80 100644 --- a/src/pkg/track/track_test.go +++ b/src/pkg/track/track_test.go @@ -2,79 +2,68 @@ package track import ( "fmt" - "strings" "testing" "github.com/stretchr/testify/assert" ) -func TestChunkMessages(t *testing.T) { - maxChunkSize := 5 +func TestEventMessages(t *testing.T) { tests := []struct { name string prefix string messages []string - expectedChunkContents []string + expectedEventContents []string }{ { name: "empty messages", prefix: "logs", messages: []string{}, - expectedChunkContents: []string{}, + expectedEventContents: []string{}, }, { name: "single message", prefix: "logs", messages: []string{"msg"}, - expectedChunkContents: []string{"msg"}, + expectedEventContents: []string{"msg"}, }, { - name: "three messages - one chunk", + name: "three messages - three events", prefix: "logs", messages: []string{"1", "2", "3"}, - expectedChunkContents: []string{"1\n2\n3"}, + expectedEventContents: []string{"1", "2", "3"}, }, { - name: "four messages - 4 chunks", + name: "long message- truncated", prefix: "logs", - messages: []string{"123", "456", "789", "abc"}, - expectedChunkContents: []string{"123\n4", "56\n78", "9\nabc"}, + messages: []string{"012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789"}, + expectedEventContents: []string{"012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234"}, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - result := ChunkMessagesWithSize(tt.prefix, tt.messages, maxChunkSize) + result := MakeEventLogProperties(tt.prefix, tt.messages) - // Check number of chunks - assert.Equal(t, len(tt.expectedChunkContents), len(result), "incorrect number of chunks") + // Check number of event properties created + assert.Equal(t, len(tt.expectedEventContents), len(result), "incorrect number of event properties") if len(tt.messages) == 0 { return // No more checks needed for empty input } - // Verify chunk sizes and property names + // Verify event properties for i, prop := range result { // Check property name format expectedName := fmt.Sprintf("%s-%d", tt.prefix, i+1) assert.Equal(t, expectedName, prop.Name, "incorrect property name") // Check that value is string - msgs, ok := prop.Value.(string) + propValue, ok := prop.Value.(string) assert.True(t, ok, "property value should be string") - // Check chunk size - if i < len(result)-1 { - // All chunks except the last should have maxMessagePerProperty messages - assert.Equal(t, maxChunkSize, len(msgs), "non-final chunk has incorrect size") - } else { - // final chunk - assert.LessOrEqual(t, len(msgs), maxChunkSize, "final chunk exceeds max size") - assert.Greater(t, len(msgs), 0, "final chunk should not be empty") - - lastValue, ok := result[len(result)-1].Value.(string) - assert.True(t, ok, "last chunk value should be string") - assert.True(t, strings.HasSuffix(msgs, lastValue), "final chunk content mismatch") + // Check size + if len(tt.messages[i]) > maxPropertyCharacterLength && len(propValue) != maxPropertyCharacterLength { + assert.Less(t, len(propValue), maxPropertyCharacterLength, "property value exceeds maxPropertyCharacterLength at %d", len(propValue)) } } }) From 6d779ad07f9edb02162a6b92812233a19b7a5da9 Mon Sep 17 00:00:00 2001 From: goreleaserbot Date: Fri, 5 Jul 2024 16:42:42 +0000 Subject: [PATCH 07/18] defang: -> v0.5.32 From 426f9f6094a10569a89c00026412011adc1fbdd2 Mon Sep 17 00:00:00 2001 From: goreleaserbot Date: Fri, 5 Jul 2024 16:54:56 +0000 Subject: [PATCH 08/18] defang: -> v0.5.32 From eab8a5c6f979d94f746c37c8367b9c04ed247bf4 Mon Sep 17 00:00:00 2001 From: goreleaserbot Date: Sun, 23 Mar 2025 20:42:37 +0000 Subject: [PATCH 09/18] defang: -> v1.1.9 From 5ddc078ded3de67e6cfca449444bd318edf7c68a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lio=E6=9D=8E=E6=AD=90?= Date: Tue, 5 Aug 2025 12:14:54 -0700 Subject: [PATCH 10/18] fix(cd): hide node warnings (#1343) fixes #1272 From b6ac6e62affd741f0728c5a2e0670ec19b96f5d5 Mon Sep 17 00:00:00 2001 From: Eric Liu Date: Tue, 28 Oct 2025 14:54:55 -0700 Subject: [PATCH 11/18] use only tail logs when capturing deploy errors --- src/cmd/cli/command/commands.go | 10 +++--- src/cmd/cli/command/compose.go | 29 +++++++++++----- src/pkg/cli/client/errors.go | 13 ++++++++ src/pkg/cli/tail.go | 6 ++++ src/pkg/datastructs/circularbuffer.go | 2 -- src/pkg/term/colorizer.go | 45 ++++++------------------- src/pkg/term/colorizer_test.go | 48 +++------------------------ src/pkg/track/track.go | 12 +++---- 8 files changed, 65 insertions(+), 100 deletions(-) diff --git a/src/cmd/cli/command/commands.go b/src/cmd/cli/command/commands.go index b1b85782a..adc630d00 100644 --- a/src/cmd/cli/command/commands.go +++ b/src/cmd/cli/command/commands.go @@ -352,11 +352,11 @@ var RootCmd = &cobra.Command{ var errString = "" logProps := []track.Property{} if err != nil { - errString = err.Error() - - // on error, also log the recent terminal messages - messages := term.DefaultTerm.GetAllMessages() - logProps = append(logProps, track.MakeEventLogProperties("log", messages)...) + var errWithLogs cliClient.ErrWithLogCache + if errors.As(err, &errWithLogs) { + // Add log cache to tracking properties + logProps = append(logProps, track.MakeEventLogProperties("log", errWithLogs.Logs)...) + } } props := []track.Property{ diff --git a/src/cmd/cli/command/compose.go b/src/cmd/cli/command/compose.go index 56a565ac4..7ed34eec0 100644 --- a/src/cmd/cli/command/compose.go +++ b/src/cmd/cli/command/compose.go @@ -17,6 +17,7 @@ import ( cliClient "github.com/DefangLabs/defang/src/pkg/cli/client" "github.com/DefangLabs/defang/src/pkg/cli/client/byoc" "github.com/DefangLabs/defang/src/pkg/cli/compose" + "github.com/DefangLabs/defang/src/pkg/datastructs" "github.com/DefangLabs/defang/src/pkg/dryrun" "github.com/DefangLabs/defang/src/pkg/logs" "github.com/DefangLabs/defang/src/pkg/modes" @@ -91,7 +92,7 @@ func makeComposeUpCmd() *cobra.Command { return err } - track.EvtWithTerm("Debug Prompted", P("loadErr", loadErr)) + track.Evt("Debug Prompted", P("loadErr", loadErr)) return cli.InteractiveDebugForClientError(ctx, client, project, loadErr) } @@ -177,7 +178,7 @@ func makeComposeUpCmd() *cobra.Command { } term.Info("Tailing logs for", tailSource, "; press Ctrl+C to detach:") - tailOptions := newTailOptionsForDeploy(deploy.Etag, since, verbose) + tailOptions := newTailOptionsForDeploy(deploy.Etag, since, verbose, true) serviceStates, err := cli.TailAndMonitor(ctx, project, provider, time.Duration(waitTimeout)*time.Second, tailOptions) if err != nil { handleTailAndMonitorErr(ctx, err, client, cli.DebugConfig{ @@ -187,7 +188,11 @@ func makeComposeUpCmd() *cobra.Command { Provider: provider, Since: since, }) - return err + + return cliClient.ErrWithLogCache{ + Err: err, + Logs: tailOptions.LogCache.Get(), + } } for _, service := range deploy.Services { @@ -245,7 +250,7 @@ func handleComposeUpErr(ctx context.Context, err error, project *compose.Project } term.Error("Error:", cliClient.PrettyError(err)) - track.EvtWithTerm("Debug Prompted", P("composeErr", err)) + track.Evt("Debug Prompted", P("composeErr", err)) return cli.InteractiveDebugForClientError(ctx, client, project, err) } @@ -260,7 +265,7 @@ func handleTailAndMonitorErr(ctx context.Context, err error, client *cliClient.G if nonInteractive { printDefangHint("To debug the deployment, do:", debugConfig.String()) } else { - track.EvtWithTerm("Debug Prompted", P("failedServices", debugConfig.FailedServices), + track.Evt("Debug Prompted", P("failedServices", debugConfig.FailedServices), P("etag", debugConfig.Deployment), P("reason", errDeploymentFailed), ) @@ -268,15 +273,15 @@ func handleTailAndMonitorErr(ctx context.Context, err error, client *cliClient.G // Call the AI debug endpoint using the original command context (not the tail ctx which is canceled) if nil != cli.InteractiveDebugDeployment(ctx, client, debugConfig) { // don't show this defang hint if debugging was successful - tailOptions := newTailOptionsForDeploy(debugConfig.Deployment, debugConfig.Since, true) + tailOptions := newTailOptionsForDeploy(debugConfig.Deployment, debugConfig.Since, true, true) printDefangHint("To see the logs of the failed service, do:", tailOptions.String()) } } } } -func newTailOptionsForDeploy(deployment string, since time.Time, verbose bool) cli.TailOptions { - return cli.TailOptions{ +func newTailOptionsForDeploy(deployment string, since time.Time, verbose bool, useLogCache bool) cli.TailOptions { + tailOpt := cli.TailOptions{ Deployment: deployment, LogType: logs.LogTypeAll, // TODO: Move this to playground provider GetDeploymentStatus @@ -294,6 +299,12 @@ func newTailOptionsForDeploy(deployment string, since time.Time, verbose bool) c Since: since, Verbose: verbose, } + + if useLogCache { + logCache := datastructs.NewCircularBuffer[string](30) + tailOpt.LogCache = &logCache + } + return tailOpt } func flushWarnings() { @@ -459,7 +470,7 @@ func makeComposeConfigCmd() *cobra.Command { return err } - track.EvtWithTerm("Debug Prompted", P("loadErr", loadErr)) + track.Evt("Debug Prompted", P("loadErr", loadErr)) return cli.InteractiveDebugForClientError(ctx, client, project, loadErr) } diff --git a/src/pkg/cli/client/errors.go b/src/pkg/cli/client/errors.go index 49c831a3b..44d2afec0 100644 --- a/src/pkg/cli/client/errors.go +++ b/src/pkg/cli/client/errors.go @@ -14,3 +14,16 @@ func (e ErrDeploymentFailed) Error() string { } return fmt.Sprintf("deployment failed%s: %s", service, e.Message) } + +type ErrWithLogCache struct { + Err error + Logs []string +} + +func (e ErrWithLogCache) Error() string { + return fmt.Sprintf("Error: %v, Logs: %v", e.Err, e.Logs) +} + +func (e ErrWithLogCache) Unwrap() error { + return e.Err +} diff --git a/src/pkg/cli/tail.go b/src/pkg/cli/tail.go index 8e6dbe31b..5b12a0cea 100644 --- a/src/pkg/cli/tail.go +++ b/src/pkg/cli/tail.go @@ -15,6 +15,7 @@ import ( "github.com/DefangLabs/defang/src/pkg" "github.com/DefangLabs/defang/src/pkg/cli/client" + "github.com/DefangLabs/defang/src/pkg/datastructs" "github.com/DefangLabs/defang/src/pkg/dryrun" "github.com/DefangLabs/defang/src/pkg/logs" "github.com/DefangLabs/defang/src/pkg/spinner" @@ -45,6 +46,7 @@ type TailDetectStopEventFunc func(eventLog *defangv1.LogEntry) error type TailOptions struct { EndEventDetectFunc TailDetectStopEventFunc // Deprecated: use Subscribe and GetDeploymentStatus instead #851 + LogCache *datastructs.CircularBuffer[string] Deployment types.ETag Filter string LogType logs.LogType @@ -377,6 +379,10 @@ func streamLogs(ctx context.Context, provider client.Provider, projectName strin } func logEntryPrintHandler(e *defangv1.LogEntry, options *TailOptions) error { + if options.LogCache != nil { + options.LogCache.Add(e.Message) + } + if options.Raw { if e.Stderr { term.Error(e.Message) diff --git a/src/pkg/datastructs/circularbuffer.go b/src/pkg/datastructs/circularbuffer.go index 2c8f8368f..6cff508b8 100644 --- a/src/pkg/datastructs/circularbuffer.go +++ b/src/pkg/datastructs/circularbuffer.go @@ -1,7 +1,5 @@ package datastructs -var DefaultBufferSize = 30 - type CircularBuffer[T any] struct { size int entries int diff --git a/src/pkg/term/colorizer.go b/src/pkg/term/colorizer.go index 7cbae12e3..fab546074 100644 --- a/src/pkg/term/colorizer.go +++ b/src/pkg/term/colorizer.go @@ -8,7 +8,6 @@ import ( "slices" "strings" - "github.com/DefangLabs/defang/src/pkg/datastructs" "github.com/muesli/termenv" "golang.org/x/term" ) @@ -23,7 +22,6 @@ type Term struct { hasDarkBg bool warnings []string - buffer datastructs.CircularBuffer[string] } var DefaultTerm = NewTerm(os.Stdin, os.Stdout, os.Stderr) @@ -58,7 +56,6 @@ func NewTerm(stdin FileReader, stdout, stderr io.Writer) *Term { stderr: stderr, out: termenv.NewOutput(stdout), err: termenv.NewOutput(stderr), - buffer: datastructs.NewCircularBuffer[string](datastructs.DefaultBufferSize), } t.hasDarkBg = t.out.HasDarkBackground() if hasTermInEnv() { @@ -73,11 +70,6 @@ func (t Term) Stdio() (FileReader, termenv.File, io.Writer) { return t.stdin, t.out.TTY(), t.err } -// GetAllMessages returns all messages currently stored in the buffer in chronological order -func (t Term) GetAllMessages() []string { - return t.buffer.Get() -} - func (t *Term) ForceColor(color bool) { if color { t.out = termenv.NewOutput(t.stdout, termenv.WithProfile(termenv.ANSI)) @@ -186,79 +178,62 @@ func ensurePrefix(s string, prefix string) string { return prefix + s } -func (t *Term) output(c Color, v ...any) (int, error) { - msg := fmt.Sprint(v...) - t.buffer.Add(msg) - return output(t.out, c, msg) -} - func (t *Term) Printc(c Color, v ...any) (int, error) { - return t.output(c, v...) + return output(t.out, c, fmt.Sprint(v...)) } func (t *Term) Print(v ...any) (int, error) { - t.buffer.Add(fmt.Sprint(v...)) return fmt.Fprint(t.out, v...) } func (t *Term) Println(v ...any) (int, error) { - text := ensureNewline(fmt.Sprintln(v...)) - t.buffer.Add(text) - return fmt.Fprint(t.out, text) + return fmt.Fprint(t.out, ensureNewline(fmt.Sprintln(v...))) } func (t *Term) Printf(format string, v ...any) (int, error) { - text := ensureNewline(fmt.Sprintf(format, v...)) - t.buffer.Add(text) - return fmt.Fprint(t.out, text) + return fmt.Fprint(t.out, ensureNewline(fmt.Sprintf(format, v...))) } func (t *Term) Debug(v ...any) (int, error) { if !t.debug { return 0, nil } - return t.output(DebugColor, ensurePrefix(fmt.Sprintln(v...), " - ")) + return output(t.out, DebugColor, ensurePrefix(fmt.Sprintln(v...), " - ")) } func (t *Term) Debugf(format string, v ...any) (int, error) { if !t.debug { return 0, nil } - s := ensureNewline(ensurePrefix(fmt.Sprintf(format, v...), " - ")) - return t.output(DebugColor, s) + return output(t.out, DebugColor, ensureNewline(ensurePrefix(fmt.Sprintf(format, v...), " - "))) } func (t *Term) Info(v ...any) (int, error) { - s := ensurePrefix(fmt.Sprintln(v...), " * ") - return t.output(InfoColor, s) + return output(t.out, InfoColor, ensurePrefix(fmt.Sprintln(v...), " * ")) } func (t *Term) Infof(format string, v ...any) (int, error) { - s := ensureNewline(ensurePrefix(fmt.Sprintf(format, v...), " * ")) - return t.output(InfoColor, s) + return output(t.out, InfoColor, ensureNewline(ensurePrefix(fmt.Sprintf(format, v...), " * "))) } func (t *Term) Warn(v ...any) (int, error) { msg := ensurePrefix(fmt.Sprintln(v...), " ! ") t.warnings = append(t.warnings, msg) - return t.output(WarnColor, msg) + return output(t.out, WarnColor, msg) } func (t *Term) Warnf(format string, v ...any) (int, error) { msg := ensureNewline(ensurePrefix(fmt.Sprintf(format, v...), " ! ")) t.warnings = append(t.warnings, msg) - return t.output(WarnColor, msg) + return output(t.out, WarnColor, msg) } func (t *Term) Error(v ...any) (int, error) { - msg := fmt.Sprintln(v...) - t.buffer.Add(msg) - return output(t.err, ErrorColor, msg) + return output(t.err, ErrorColor, fmt.Sprintln(v...)) } func (t *Term) Errorf(format string, v ...any) (int, error) { line := ensureNewline(fmt.Sprintf(format, v...)) - t.buffer.Add(line) return output(t.err, ErrorColor, line) } diff --git a/src/pkg/term/colorizer_test.go b/src/pkg/term/colorizer_test.go index 226861650..2ce889179 100644 --- a/src/pkg/term/colorizer_test.go +++ b/src/pkg/term/colorizer_test.go @@ -8,9 +8,7 @@ import ( "strings" "testing" - "github.com/DefangLabs/defang/src/pkg/datastructs" "github.com/muesli/termenv" - "github.com/stretchr/testify/assert" ) func TestOutput(t *testing.T) { @@ -220,13 +218,13 @@ func TestFlushWarnings(t *testing.T) { for i, test := range tests { t.Run(strconv.Itoa(i), func(t *testing.T) { var stdout, stderr bytes.Buffer - termWriter := NewTerm(os.Stdin, &stdout, &stderr) + term := NewTerm(os.Stdin, &stdout, &stderr) for _, warning := range test.warnings { - termWriter.Warn(warning) + term.Warn(warning) } - bytesWritten, err := termWriter.FlushWarnings() + bytesWritten, err := term.FlushWarnings() if (err != nil) != test.expectErr { t.Errorf("FlushWarnings() error = %v, expectErr %v", err, test.expectErr) } @@ -239,45 +237,9 @@ func TestFlushWarnings(t *testing.T) { t.Errorf("FlushWarnings() expected %d byteWritten, got %d", bytesInExpected, bytesWritten) } - if termWriter.getAllWarnings() != nil { - t.Errorf("after FlushWarnings() expected no warnings, got %v", termWriter.getAllWarnings()) + if term.getAllWarnings() != nil { + t.Errorf("after FlushWarnings() expected no warnings, got %v", term.getAllWarnings()) } }) } } - -func TestWriteToBuffer(t *testing.T) { - var stdout, stderr bytes.Buffer - - originalBufferSize := datastructs.DefaultBufferSize - t.Cleanup(func() { - datastructs.DefaultBufferSize = originalBufferSize - }) - datastructs.DefaultBufferSize = 5 - - termWriter := NewTerm(os.Stdin, &stdout, &stderr) - - // no messages initially - currentMessages := termWriter.GetAllMessages() - assert.Empty(t, currentMessages) - - // add messages, less than buffer capacity - termWriter.Info("message 1") - termWriter.Info("message 2") - termWriter.Info("message 3") - termWriter.Info("message 4") - - // sanity check - currentMessages = termWriter.GetAllMessages() - expectedMessages := []string{" * message 1\n", " * message 2\n", " * message 3\n", " * message 4\n"} - assert.Equal(t, expectedMessages, currentMessages) - - // add messages to be over buffer capacity - termWriter.Info("message A") - termWriter.Info("message B") - - // check only the last number of message are kept - currentMessages = termWriter.GetAllMessages() - expectedMessages = []string{" * message 2\n", " * message 3\n", " * message 4\n", " * message A\n", " * message B\n"} - assert.Equal(t, expectedMessages, currentMessages) -} diff --git a/src/pkg/track/track.go b/src/pkg/track/track.go index 51c3b2053..b5a73a216 100644 --- a/src/pkg/track/track.go +++ b/src/pkg/track/track.go @@ -85,12 +85,12 @@ func MakeEventLogProperties(name string, message []string) []Property { return trackMsg } -func EvtWithTerm(eventName string, extraProps ...Property) { - messages := term.DefaultTerm.GetAllMessages() - logProps := MakeEventLogProperties(logPropertyNamePrefix, messages) - allProps := append(extraProps, logProps...) - Evt(eventName, allProps...) -} +// func EvtWithTerm(eventName string, extraProps ...Property) { +// messages := term.DefaultTerm.GetAllMessages() +// logProps := MakeEventLogProperties(logPropertyNamePrefix, messages) +// allProps := append(extraProps, logProps...) +// Evt(eventName, allProps...) +// } func isCompletionCommand(cmd *cobra.Command) bool { return cmd.Name() == cobra.ShellCompRequestCmd || (cmd.Parent() != nil && cmd.Parent().Name() == "completion") From 86b8c90b9e50464d04e880e468442c22049f6674 Mon Sep 17 00:00:00 2001 From: Eric Liu Date: Tue, 28 Oct 2025 15:45:18 -0700 Subject: [PATCH 12/18] clean up --- src/cmd/cli/command/commands.go | 15 +++------------ src/cmd/cli/command/compose.go | 23 +++++++++++++++++------ src/pkg/cli/client/errors.go | 6 +++--- 3 files changed, 23 insertions(+), 21 deletions(-) diff --git a/src/cmd/cli/command/commands.go b/src/cmd/cli/command/commands.go index adc630d00..ba2218bde 100644 --- a/src/cmd/cli/command/commands.go +++ b/src/cmd/cli/command/commands.go @@ -298,9 +298,9 @@ func SetupCommands(ctx context.Context, version string) { // MCP Command mcpCmd.AddCommand(mcpSetupCmd) + mcpServerCmd.Flags().Int("auth-server", 0, "auth server port") mcpCmd.AddCommand(mcpServerCmd) mcpCmd.PersistentFlags().String("client", "", fmt.Sprintf("MCP setup client %v", mcp.ValidClients)) - mcpServerCmd.Flags().Int("auth-server", 0, "auth server port") RootCmd.AddCommand(mcpCmd) // Send Command @@ -350,20 +350,11 @@ var RootCmd = &cobra.Command{ // Use "defer" to track any errors that occur during the command defer func() { var errString = "" - logProps := []track.Property{} if err != nil { - var errWithLogs cliClient.ErrWithLogCache - if errors.As(err, &errWithLogs) { - // Add log cache to tracking properties - logProps = append(logProps, track.MakeEventLogProperties("log", errWithLogs.Logs)...) - } + errString = err.Error() } - props := []track.Property{ - P("args", args), P("err", errString), P("non-interactive", nonInteractive), P("provider", providerID), - } - props = append(props, logProps...) - track.Cmd(cmd, "Invoked", props...) + track.Cmd(cmd, "Invoked", P("args", args), P("err", errString), P("non-interactive", nonInteractive), P("provider", providerID)) }() // Do this first, since any errors will be printed to the console diff --git a/src/cmd/cli/command/compose.go b/src/cmd/cli/command/compose.go index 7ed34eec0..3992b40a3 100644 --- a/src/cmd/cli/command/compose.go +++ b/src/cmd/cli/command/compose.go @@ -181,7 +181,12 @@ func makeComposeUpCmd() *cobra.Command { tailOptions := newTailOptionsForDeploy(deploy.Etag, since, verbose, true) serviceStates, err := cli.TailAndMonitor(ctx, project, provider, time.Duration(waitTimeout)*time.Second, tailOptions) if err != nil { - handleTailAndMonitorErr(ctx, err, client, cli.DebugConfig{ + errWithLog := cliClient.ErrWithLogs{ + Err: err, + Logs: tailOptions.LogCache.Get(), + } + + handleTailAndMonitorErr(ctx, errWithLog, client, cli.DebugConfig{ Deployment: deploy.Etag, ModelId: modelId, Project: project, @@ -189,10 +194,7 @@ func makeComposeUpCmd() *cobra.Command { Since: since, }) - return cliClient.ErrWithLogCache{ - Err: err, - Logs: tailOptions.LogCache.Get(), - } + return err } for _, service := range deploy.Services { @@ -265,10 +267,19 @@ func handleTailAndMonitorErr(ctx context.Context, err error, client *cliClient.G if nonInteractive { printDefangHint("To debug the deployment, do:", debugConfig.String()) } else { - track.Evt("Debug Prompted", P("failedServices", debugConfig.FailedServices), + props := []track.Property{} + var errWithLog cliClient.ErrWithLogs + if errors.As(err, &errWithLog) { + props = track.MakeEventLogProperties("logs", errWithLog.Logs) + } + + props = append(props, + P("failedServices", debugConfig.FailedServices), P("etag", debugConfig.Deployment), P("reason", errDeploymentFailed), ) + track.Evt("Debug Prompted", props..., + ) // Call the AI debug endpoint using the original command context (not the tail ctx which is canceled) if nil != cli.InteractiveDebugDeployment(ctx, client, debugConfig) { diff --git a/src/pkg/cli/client/errors.go b/src/pkg/cli/client/errors.go index 44d2afec0..1bf059761 100644 --- a/src/pkg/cli/client/errors.go +++ b/src/pkg/cli/client/errors.go @@ -15,15 +15,15 @@ func (e ErrDeploymentFailed) Error() string { return fmt.Sprintf("deployment failed%s: %s", service, e.Message) } -type ErrWithLogCache struct { +type ErrWithLogs struct { Err error Logs []string } -func (e ErrWithLogCache) Error() string { +func (e ErrWithLogs) Error() string { return fmt.Sprintf("Error: %v, Logs: %v", e.Err, e.Logs) } -func (e ErrWithLogCache) Unwrap() error { +func (e ErrWithLogs) Unwrap() error { return e.Err } From ece98f150cea7b19b311719d5f8cb8a6697d1981 Mon Sep 17 00:00:00 2001 From: Eric Liu Date: Tue, 28 Oct 2025 22:13:55 -0700 Subject: [PATCH 13/18] remove new error type --- src/cmd/cli/command/compose.go | 16 +++++++--------- src/pkg/cli/client/errors.go | 13 ------------- 2 files changed, 7 insertions(+), 22 deletions(-) diff --git a/src/cmd/cli/command/compose.go b/src/cmd/cli/command/compose.go index 5f0e54524..c10148934 100644 --- a/src/cmd/cli/command/compose.go +++ b/src/cmd/cli/command/compose.go @@ -154,12 +154,11 @@ func makeComposeUpCmd() *cobra.Command { tailOptions := newTailOptionsForDeploy(deploy.Etag, since, verbose, true) serviceStates, err := cli.TailAndMonitor(ctx, project, provider, time.Duration(waitTimeout)*time.Second, tailOptions) if err != nil { - errWithLog := cliClient.ErrWithLogs{ - Err: err, - Logs: tailOptions.LogCache.Get(), + logs := []string{} + if tailOptions.LogCache != nil { + logs = tailOptions.LogCache.Get() } - - handleTailAndMonitorErr(ctx, errWithLog, client, cli.DebugConfig{ + handleTailAndMonitorErr(ctx, err, &logs, client, cli.DebugConfig{ Deployment: deploy.Etag, ModelId: modelId, Project: project, @@ -229,7 +228,7 @@ func handleComposeUpErr(ctx context.Context, err error, project *compose.Project return cli.InteractiveDebugForClientError(ctx, client, project, err) } -func handleTailAndMonitorErr(ctx context.Context, err error, client *cliClient.GrpcClient, debugConfig cli.DebugConfig) { +func handleTailAndMonitorErr(ctx context.Context, err error, logs *[]string, client *cliClient.GrpcClient, debugConfig cli.DebugConfig) { var errDeploymentFailed cliClient.ErrDeploymentFailed if errors.As(err, &errDeploymentFailed) { // Tail got canceled because of deployment failure: prompt to show the debugger @@ -241,9 +240,8 @@ func handleTailAndMonitorErr(ctx context.Context, err error, client *cliClient.G printDefangHint("To debug the deployment, do:", debugConfig.String()) } else { props := []track.Property{} - var errWithLog cliClient.ErrWithLogs - if errors.As(err, &errWithLog) { - props = track.MakeEventLogProperties("logs", errWithLog.Logs) + if logs != nil { + props = track.MakeEventLogProperties("logs", *logs) } props = append(props, diff --git a/src/pkg/cli/client/errors.go b/src/pkg/cli/client/errors.go index 1bf059761..49c831a3b 100644 --- a/src/pkg/cli/client/errors.go +++ b/src/pkg/cli/client/errors.go @@ -14,16 +14,3 @@ func (e ErrDeploymentFailed) Error() string { } return fmt.Sprintf("deployment failed%s: %s", service, e.Message) } - -type ErrWithLogs struct { - Err error - Logs []string -} - -func (e ErrWithLogs) Error() string { - return fmt.Sprintf("Error: %v, Logs: %v", e.Err, e.Logs) -} - -func (e ErrWithLogs) Unwrap() error { - return e.Err -} From 41dadd4f1b345f3d931d2886eaa43854565d4ba3 Mon Sep 17 00:00:00 2001 From: Eric Liu Date: Tue, 28 Oct 2025 22:24:42 -0700 Subject: [PATCH 14/18] remove unecessary compose tailoption flag --- src/cmd/cli/command/compose.go | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/src/cmd/cli/command/compose.go b/src/cmd/cli/command/compose.go index c10148934..107193fba 100644 --- a/src/cmd/cli/command/compose.go +++ b/src/cmd/cli/command/compose.go @@ -151,7 +151,7 @@ func makeComposeUpCmd() *cobra.Command { } term.Info("Tailing logs for", tailSource, "; press Ctrl+C to detach:") - tailOptions := newTailOptionsForDeploy(deploy.Etag, since, verbose, true) + tailOptions := newTailOptionsForDeploy(deploy.Etag, since, verbose) serviceStates, err := cli.TailAndMonitor(ctx, project, provider, time.Duration(waitTimeout)*time.Second, tailOptions) if err != nil { logs := []string{} @@ -255,14 +255,14 @@ func handleTailAndMonitorErr(ctx context.Context, err error, logs *[]string, cli // Call the AI debug endpoint using the original command context (not the tail ctx which is canceled) if nil != cli.InteractiveDebugDeployment(ctx, client, debugConfig) { // don't show this defang hint if debugging was successful - tailOptions := newTailOptionsForDeploy(debugConfig.Deployment, debugConfig.Since, true, true) + tailOptions := newTailOptionsForDeploy(debugConfig.Deployment, debugConfig.Since, true) printDefangHint("To see the logs of the failed service, do:", tailOptions.String()) } } } } -func newTailOptionsForDeploy(deployment string, since time.Time, verbose bool, useLogCache bool) cli.TailOptions { +func newTailOptionsForDeploy(deployment string, since time.Time, verbose bool) cli.TailOptions { tailOpt := cli.TailOptions{ Deployment: deployment, LogType: logs.LogTypeAll, @@ -282,10 +282,8 @@ func newTailOptionsForDeploy(deployment string, since time.Time, verbose bool, u Verbose: verbose, } - if useLogCache { - logCache := datastructs.NewCircularBuffer[string](30) - tailOpt.LogCache = &logCache - } + logCache := datastructs.NewCircularBuffer[string](30) + tailOpt.LogCache = &logCache return tailOpt } From ecfcba1780fd9ac32bdc659fc96232da43368c42 Mon Sep 17 00:00:00 2001 From: Eric Liu Date: Fri, 31 Oct 2025 11:52:40 -0700 Subject: [PATCH 15/18] Apply suggestions from code review Co-authored-by: Jordan Stephens --- src/cmd/cli/command/compose.go | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/cmd/cli/command/compose.go b/src/cmd/cli/command/compose.go index 107193fba..75f5bb2f5 100644 --- a/src/cmd/cli/command/compose.go +++ b/src/cmd/cli/command/compose.go @@ -158,7 +158,7 @@ func makeComposeUpCmd() *cobra.Command { if tailOptions.LogCache != nil { logs = tailOptions.LogCache.Get() } - handleTailAndMonitorErr(ctx, err, &logs, client, cli.DebugConfig{ + handleTailAndMonitorErr(ctx, err, logs, client, cli.DebugConfig{ Deployment: deploy.Etag, ModelId: modelId, Project: project, @@ -228,7 +228,7 @@ func handleComposeUpErr(ctx context.Context, err error, project *compose.Project return cli.InteractiveDebugForClientError(ctx, client, project, err) } -func handleTailAndMonitorErr(ctx context.Context, err error, logs *[]string, client *cliClient.GrpcClient, debugConfig cli.DebugConfig) { +func handleTailAndMonitorErr(ctx context.Context, err error, logs []string, client *cliClient.GrpcClient, debugConfig cli.DebugConfig) { var errDeploymentFailed cliClient.ErrDeploymentFailed if errors.As(err, &errDeploymentFailed) { // Tail got canceled because of deployment failure: prompt to show the debugger @@ -240,9 +240,7 @@ func handleTailAndMonitorErr(ctx context.Context, err error, logs *[]string, cli printDefangHint("To debug the deployment, do:", debugConfig.String()) } else { props := []track.Property{} - if logs != nil { - props = track.MakeEventLogProperties("logs", *logs) - } + props = track.MakeEventLogProperties("logs", logs) props = append(props, P("failedServices", debugConfig.FailedServices), From 10a113523e4522cbabaefa222919582c3bbc7d2a Mon Sep 17 00:00:00 2001 From: Eric Liu Date: Fri, 31 Oct 2025 13:31:55 -0700 Subject: [PATCH 16/18] review updates --- src/cmd/cli/command/compose.go | 16 ++++++---------- src/pkg/cli/bootstrap.go | 4 +++- src/pkg/cli/composeUp_test.go | 2 +- src/pkg/cli/estimate.go | 4 +++- src/pkg/cli/tail.go | 17 ++++++++--------- src/pkg/cli/tailAndMonitor.go | 10 ++++++---- src/pkg/cli/tail_test.go | 10 +++++----- src/pkg/datastructs/circularbuffer.go | 6 ++++++ src/pkg/mcp/tools/default_tool_cli.go | 3 ++- src/pkg/mcp/tools/logs.go | 5 +++-- 10 files changed, 43 insertions(+), 34 deletions(-) diff --git a/src/cmd/cli/command/compose.go b/src/cmd/cli/command/compose.go index 75f5bb2f5..2f1a5f2b6 100644 --- a/src/cmd/cli/command/compose.go +++ b/src/cmd/cli/command/compose.go @@ -15,7 +15,6 @@ import ( cliClient "github.com/DefangLabs/defang/src/pkg/cli/client" "github.com/DefangLabs/defang/src/pkg/cli/client/byoc" "github.com/DefangLabs/defang/src/pkg/cli/compose" - "github.com/DefangLabs/defang/src/pkg/datastructs" "github.com/DefangLabs/defang/src/pkg/dryrun" "github.com/DefangLabs/defang/src/pkg/logs" "github.com/DefangLabs/defang/src/pkg/modes" @@ -152,13 +151,10 @@ func makeComposeUpCmd() *cobra.Command { term.Info("Tailing logs for", tailSource, "; press Ctrl+C to detach:") tailOptions := newTailOptionsForDeploy(deploy.Etag, since, verbose) - serviceStates, err := cli.TailAndMonitor(ctx, project, provider, time.Duration(waitTimeout)*time.Second, tailOptions) + serviceStates, logCache, err := cli.TailAndMonitor(ctx, project, provider, time.Duration(waitTimeout)*time.Second, tailOptions) if err != nil { - logs := []string{} - if tailOptions.LogCache != nil { - logs = tailOptions.LogCache.Get() - } - handleTailAndMonitorErr(ctx, err, logs, client, cli.DebugConfig{ + logs := logCache.Get() + handleTailAndMonitorErr(ctx, err, &logs, client, cli.DebugConfig{ Deployment: deploy.Etag, ModelId: modelId, Project: project, @@ -280,8 +276,6 @@ func newTailOptionsForDeploy(deployment string, since time.Time, verbose bool) c Verbose: verbose, } - logCache := datastructs.NewCircularBuffer[string](30) - tailOpt.LogCache = &logCache return tailOpt } @@ -596,7 +590,9 @@ func makeComposeLogsCmd() *cobra.Command { Until: untilTs, Verbose: verbose, } - return cli.Tail(cmd.Context(), provider, projectName, tailOptions) + + _, err = cli.Tail(cmd.Context(), provider, projectName, tailOptions) + return err }, } logsCmd.Flags().StringP("name", "n", "", "name of the service (backwards compat)") diff --git a/src/pkg/cli/bootstrap.go b/src/pkg/cli/bootstrap.go index d02303224..50705d286 100644 --- a/src/pkg/cli/bootstrap.go +++ b/src/pkg/cli/bootstrap.go @@ -9,6 +9,7 @@ import ( "github.com/DefangLabs/defang/src/pkg" "github.com/DefangLabs/defang/src/pkg/cli/client" + "github.com/DefangLabs/defang/src/pkg/datastructs" "github.com/DefangLabs/defang/src/pkg/dryrun" "github.com/DefangLabs/defang/src/pkg/logs" "github.com/DefangLabs/defang/src/pkg/term" @@ -50,9 +51,10 @@ func TailAndWaitForCD(ctx context.Context, provider client.Provider, projectName cancelTail(cdErr) }() + logCache := datastructs.NewCircularBuffer[string](30) // blocking call to tail var tailErr error - if err := streamLogs(ctx, provider, projectName, tailOptions, logEntryPrintHandler); err != nil { + if err := streamLogs(ctx, provider, projectName, tailOptions, logCache, logEntryPrintHandler); err != nil { term.Debug("Tail stopped with", err, errors.Unwrap(err)) if !errors.Is(err, context.Canceled) { tailErr = err diff --git a/src/pkg/cli/composeUp_test.go b/src/pkg/cli/composeUp_test.go index fb873faac..6e44ed4d4 100644 --- a/src/pkg/cli/composeUp_test.go +++ b/src/pkg/cli/composeUp_test.go @@ -291,7 +291,7 @@ func TestComposeUpStops(t *testing.T) { timer := time.AfterFunc(time.Second, func() { provider.subscribeStream.Send(tt.svcFailed, tt.subscribeErr) }) t.Cleanup(func() { timer.Stop() }) } - _, err = TailAndMonitor(ctx, project, provider, -1, TailOptions{Deployment: resp.Etag}) + _, _, err = TailAndMonitor(ctx, project, provider, -1, TailOptions{Deployment: resp.Etag}) if err != nil { if err.Error() != tt.wantError { t.Errorf("expected error: %v, got: %v", tt.wantError, err) diff --git a/src/pkg/cli/estimate.go b/src/pkg/cli/estimate.go index e229f9ab5..67b2da98e 100644 --- a/src/pkg/cli/estimate.go +++ b/src/pkg/cli/estimate.go @@ -13,6 +13,7 @@ import ( "github.com/DefangLabs/defang/src/pkg/cli/client" "github.com/DefangLabs/defang/src/pkg/cli/compose" + "github.com/DefangLabs/defang/src/pkg/datastructs" "github.com/DefangLabs/defang/src/pkg/logs" "github.com/DefangLabs/defang/src/pkg/modes" "github.com/DefangLabs/defang/src/pkg/money" @@ -76,7 +77,8 @@ func GeneratePreview(ctx context.Context, project *compose.Project, client clien Verbose: true, } - err = streamLogs(ctx, previewProvider, project.Name, tailOptions, func(entry *defangv1.LogEntry, options *TailOptions) error { + logCache := datastructs.NewCircularBuffer[string](30) + err = streamLogs(ctx, previewProvider, project.Name, tailOptions, logCache, func(entry *defangv1.LogEntry, options *TailOptions) error { if strings.HasPrefix(entry.Message, "Preview succeeded") { return io.EOF } else if strings.HasPrefix(entry.Message, "Preview failed") { diff --git a/src/pkg/cli/tail.go b/src/pkg/cli/tail.go index f5ae28c8e..89c60ed80 100644 --- a/src/pkg/cli/tail.go +++ b/src/pkg/cli/tail.go @@ -46,7 +46,6 @@ type TailDetectStopEventFunc func(eventLog *defangv1.LogEntry) error type TailOptions struct { EndEventDetectFunc TailDetectStopEventFunc // Deprecated: use Subscribe and GetDeploymentStatus instead #851 - LogCache *datastructs.CircularBuffer[string] Deployment types.ETag Filter string LogType logs.LogType @@ -141,7 +140,9 @@ func (cerr CancelError) Unwrap() error { return cerr.error } -func Tail(ctx context.Context, provider client.Provider, projectName string, options TailOptions) error { +func Tail(ctx context.Context, provider client.Provider, projectName string, options TailOptions) (datastructs.BufferInterface[string], error) { + logCache := datastructs.NewCircularBuffer[string](30) + if options.LogType == logs.LogTypeUnspecified { options.LogType = logs.LogTypeAll } @@ -165,10 +166,10 @@ func Tail(ctx context.Context, provider client.Provider, projectName string, opt } if dryrun.DoDryRun { - return dryrun.ErrDryRun + return &logCache, dryrun.ErrDryRun } - return streamLogs(ctx, provider, projectName, options, logEntryPrintHandler) + return &logCache, streamLogs(ctx, provider, projectName, options, logCache, logEntryPrintHandler) } func isTransientError(err error) bool { @@ -204,7 +205,7 @@ func isTransientError(err error) bool { type LogEntryHandler func(*defangv1.LogEntry, *TailOptions) error -func streamLogs(ctx context.Context, provider client.Provider, projectName string, options TailOptions, handler LogEntryHandler) error { +func streamLogs(ctx context.Context, provider client.Provider, projectName string, options TailOptions, logCache datastructs.CircularBuffer[string], handler LogEntryHandler) error { var sinceTs, untilTs *timestamppb.Timestamp if pkg.IsValidTime(options.Since) { sinceTs = timestamppb.New(options.Since) @@ -354,6 +355,8 @@ func streamLogs(ctx context.Context, provider client.Provider, projectName strin host := e.Host service := e.Service + logCache.Add(e.Message) + // HACK: skip noisy CI/CD logs (except errors) isInternal := service == "cd" || service == "kaniko" || service == "fabric" || host == "kaniko" || host == "fabric" || host == "ecs" || host == "cloudbuild" || host == "pulumi" onlyErrors := !options.Verbose && isInternal @@ -381,10 +384,6 @@ func streamLogs(ctx context.Context, provider client.Provider, projectName strin } func logEntryPrintHandler(e *defangv1.LogEntry, options *TailOptions) error { - if options.LogCache != nil { - options.LogCache.Add(e.Message) - } - if options.Raw { if e.Stderr { term.Error(e.Message) diff --git a/src/pkg/cli/tailAndMonitor.go b/src/pkg/cli/tailAndMonitor.go index cbc4bafb2..5292a0334 100644 --- a/src/pkg/cli/tailAndMonitor.go +++ b/src/pkg/cli/tailAndMonitor.go @@ -10,6 +10,7 @@ import ( "github.com/DefangLabs/defang/src/pkg" "github.com/DefangLabs/defang/src/pkg/cli/client" "github.com/DefangLabs/defang/src/pkg/cli/compose" + "github.com/DefangLabs/defang/src/pkg/datastructs" "github.com/DefangLabs/defang/src/pkg/term" defangv1 "github.com/DefangLabs/defang/src/protos/io/defang/v1" "github.com/bufbuild/connect-go" @@ -17,7 +18,7 @@ import ( const targetServiceState = defangv1.ServiceState_DEPLOYMENT_COMPLETED -func TailAndMonitor(ctx context.Context, project *compose.Project, provider client.Provider, waitTimeout time.Duration, tailOptions TailOptions) (ServiceStates, error) { +func TailAndMonitor(ctx context.Context, project *compose.Project, provider client.Provider, waitTimeout time.Duration, tailOptions TailOptions) (ServiceStates, datastructs.BufferInterface[string], error) { if tailOptions.Deployment == "" { panic("tailOptions.Deployment must be a valid deployment ID") } @@ -66,8 +67,9 @@ func TailAndMonitor(ctx context.Context, project *compose.Project, provider clie }() // blocking call to tail - var tailErr error - if err := Tail(tailCtx, provider, project.Name, tailOptions); err != nil { + var err, tailErr error + var logCache datastructs.BufferInterface[string] + if logCache, err = Tail(tailCtx, provider, project.Name, tailOptions); err != nil { term.Debug("Tail stopped with", err, errors.Unwrap(err)) if connect.CodeOf(err) == connect.CodePermissionDenied { @@ -98,7 +100,7 @@ func TailAndMonitor(ctx context.Context, project *compose.Project, provider clie } } - return serviceStates, errors.Join(cdErr, svcErr, tailErr) + return serviceStates, logCache, errors.Join(cdErr, svcErr, tailErr) } func CanMonitorService(service compose.ServiceConfig) bool { diff --git a/src/pkg/cli/tail_test.go b/src/pkg/cli/tail_test.go index 684b8a825..662a2a222 100644 --- a/src/pkg/cli/tail_test.go +++ b/src/pkg/cli/tail_test.go @@ -150,7 +150,7 @@ func TestTail(t *testing.T) { }, } - err := Tail(t.Context(), p, projectName, TailOptions{Verbose: true}) // Output host + _, err := Tail(t.Context(), p, projectName, TailOptions{Verbose: true}) // Output host if err != io.EOF { t.Errorf("Tail() error = %v, want io.EOF", err) } @@ -249,7 +249,7 @@ func TestUTC(t *testing.T) { localMock = localMock.MockTimestamp(localTime) // Start the terminal for local time test - err := Tail(t.Context(), localMock, projectName, TailOptions{Verbose: true}) // Output host + _, err := Tail(t.Context(), localMock, projectName, TailOptions{Verbose: true}) // Output host if err != nil { t.Errorf("Tail() error = %v, want io.EOF", err) } @@ -281,7 +281,7 @@ func TestUTC(t *testing.T) { utcMock := &mockTailProvider{} utcMock = utcMock.MockTimestamp(utcTime) - err = Tail(t.Context(), utcMock, projectName, TailOptions{Verbose: true}) + _, err = Tail(t.Context(), utcMock, projectName, TailOptions{Verbose: true}) if err != nil { t.Errorf("Tail() error = %v, want io.EOF", err) } @@ -330,7 +330,7 @@ func TestTailError(t *testing.T) { mock := &mockQueryErrorProvider{ TailStreamError: tt.err, } - err := Tail(t.Context(), mock, "project", tailOptions) + _, err := Tail(t.Context(), mock, "project", tailOptions) if err != nil { if err.Error() != tt.wantError { t.Errorf("Tail() error = %q, want: %q", err.Error(), tt.wantError) @@ -366,7 +366,7 @@ func TestTailContext(t *testing.T) { time.AfterFunc(10*time.Millisecond, func() { mock.tailStream.Send(nil, tt.cause) }) - err := Tail(ctx, mock, "project", tailOptions) + _, err := Tail(ctx, mock, "project", tailOptions) if err.Error() != tt.wantError { t.Errorf("Tail() error = %q, want: %q", err.Error(), tt.wantError) } diff --git a/src/pkg/datastructs/circularbuffer.go b/src/pkg/datastructs/circularbuffer.go index 6cff508b8..5a24abd77 100644 --- a/src/pkg/datastructs/circularbuffer.go +++ b/src/pkg/datastructs/circularbuffer.go @@ -1,5 +1,11 @@ package datastructs +// BufferInterface abstracts the buffer operations +type BufferInterface[T any] interface { + Add(item T) + Get() []T +} + type CircularBuffer[T any] struct { size int entries int diff --git a/src/pkg/mcp/tools/default_tool_cli.go b/src/pkg/mcp/tools/default_tool_cli.go index 5a4340f86..999849ffb 100644 --- a/src/pkg/mcp/tools/default_tool_cli.go +++ b/src/pkg/mcp/tools/default_tool_cli.go @@ -10,6 +10,7 @@ import ( "github.com/DefangLabs/defang/src/pkg/cli" cliClient "github.com/DefangLabs/defang/src/pkg/cli/client" "github.com/DefangLabs/defang/src/pkg/cli/compose" + "github.com/DefangLabs/defang/src/pkg/datastructs" "github.com/DefangLabs/defang/src/pkg/login" "github.com/DefangLabs/defang/src/pkg/mcp/common" "github.com/DefangLabs/defang/src/pkg/mcp/deployment_info" @@ -56,7 +57,7 @@ func (DefaultToolCLI) ComposeUp(ctx context.Context, project *compose.Project, c return cli.ComposeUp(ctx, project, client, provider, uploadMode, mode) } -func (DefaultToolCLI) Tail(ctx context.Context, provider cliClient.Provider, project *compose.Project, options cli.TailOptions) error { +func (DefaultToolCLI) Tail(ctx context.Context, provider cliClient.Provider, project *compose.Project, options cli.TailOptions) (datastructs.BufferInterface[string], error) { return cli.Tail(ctx, provider, project.Name, options) } diff --git a/src/pkg/mcp/tools/logs.go b/src/pkg/mcp/tools/logs.go index f10468da2..ffcc09196 100644 --- a/src/pkg/mcp/tools/logs.go +++ b/src/pkg/mcp/tools/logs.go @@ -8,6 +8,7 @@ import ( cliTypes "github.com/DefangLabs/defang/src/pkg/cli" cliClient "github.com/DefangLabs/defang/src/pkg/cli/client" "github.com/DefangLabs/defang/src/pkg/cli/compose" + "github.com/DefangLabs/defang/src/pkg/datastructs" "github.com/DefangLabs/defang/src/pkg/term" "github.com/mark3labs/mcp-go/mcp" ) @@ -52,7 +53,7 @@ type LogsCLIInterface interface { connecter providerFactory // Unique methods - Tail(ctx context.Context, provider cliClient.Provider, project *compose.Project, options cliTypes.TailOptions) error + Tail(ctx context.Context, provider cliClient.Provider, project *compose.Project, options cliTypes.TailOptions) (datastructs.BufferInterface[string], error) CheckProviderConfigured(ctx context.Context, client *cliClient.GrpcClient, providerId cliClient.ProviderID, projectName string, serviceCount int) (cliClient.Provider, error) LoadProject(ctx context.Context, loader cliClient.Loader) (*compose.Project, error) } @@ -80,7 +81,7 @@ func handleLogsTool(ctx context.Context, loader cliClient.ProjectLoader, params return "", fmt.Errorf("provider not configured correctly: %w", err) } - err = cli.Tail(ctx, provider, project, cliTypes.TailOptions{ + _, err = cli.Tail(ctx, provider, project, cliTypes.TailOptions{ Deployment: params.DeploymentID, Since: params.Since, Until: params.Until, From df56af01eb327ef3c13879994023d0d3078d05c5 Mon Sep 17 00:00:00 2001 From: Eric Liu Date: Fri, 31 Oct 2025 13:47:47 -0700 Subject: [PATCH 17/18] Update src/pkg/track/track.go MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Lio李歐 --- src/pkg/track/track.go | 7 ------- 1 file changed, 7 deletions(-) diff --git a/src/pkg/track/track.go b/src/pkg/track/track.go index b5a73a216..115e72837 100644 --- a/src/pkg/track/track.go +++ b/src/pkg/track/track.go @@ -85,13 +85,6 @@ func MakeEventLogProperties(name string, message []string) []Property { return trackMsg } -// func EvtWithTerm(eventName string, extraProps ...Property) { -// messages := term.DefaultTerm.GetAllMessages() -// logProps := MakeEventLogProperties(logPropertyNamePrefix, messages) -// allProps := append(extraProps, logProps...) -// Evt(eventName, allProps...) -// } - func isCompletionCommand(cmd *cobra.Command) bool { return cmd.Name() == cobra.ShellCompRequestCmd || (cmd.Parent() != nil && cmd.Parent().Name() == "completion") } From 3c07fdab9dd3c659d1a62dd5973ad3f033450473 Mon Sep 17 00:00:00 2001 From: Eric Liu Date: Fri, 31 Oct 2025 16:29:48 -0700 Subject: [PATCH 18/18] review update --- src/cmd/cli/command/compose.go | 9 +++---- .../circularbuffer.go | 10 +++---- .../circularbuffer_test.go | 2 +- src/pkg/cli/bootstrap.go | 4 +-- src/pkg/cli/estimate.go | 4 +-- src/pkg/cli/tail.go | 27 +++++++++---------- src/pkg/cli/tailAndMonitor.go | 6 ++--- src/pkg/mcp/tools/default_tool_cli.go | 4 +-- src/pkg/mcp/tools/logs.go | 4 +-- src/pkg/track/track.go | 5 ++-- 10 files changed, 34 insertions(+), 41 deletions(-) rename src/pkg/{datastructs => circularbuffer}/circularbuffer.go (84%) rename src/pkg/{datastructs => circularbuffer}/circularbuffer_test.go (95%) diff --git a/src/cmd/cli/command/compose.go b/src/cmd/cli/command/compose.go index 2f1a5f2b6..6462110a9 100644 --- a/src/cmd/cli/command/compose.go +++ b/src/cmd/cli/command/compose.go @@ -154,7 +154,7 @@ func makeComposeUpCmd() *cobra.Command { serviceStates, logCache, err := cli.TailAndMonitor(ctx, project, provider, time.Duration(waitTimeout)*time.Second, tailOptions) if err != nil { logs := logCache.Get() - handleTailAndMonitorErr(ctx, err, &logs, client, cli.DebugConfig{ + handleTailAndMonitorErr(ctx, err, logs, client, cli.DebugConfig{ Deployment: deploy.Etag, ModelId: modelId, Project: project, @@ -235,16 +235,13 @@ func handleTailAndMonitorErr(ctx context.Context, err error, logs []string, clie if nonInteractive { printDefangHint("To debug the deployment, do:", debugConfig.String()) } else { - props := []track.Property{} - props = track.MakeEventLogProperties("logs", logs) - + props := track.MakeEventLogProperties("logs", logs) props = append(props, P("failedServices", debugConfig.FailedServices), P("etag", debugConfig.Deployment), P("reason", errDeploymentFailed), ) - track.Evt("Debug Prompted", props..., - ) + track.Evt("Debug Prompted", props...) // Call the AI debug endpoint using the original command context (not the tail ctx which is canceled) if nil != cli.InteractiveDebugDeployment(ctx, client, debugConfig) { diff --git a/src/pkg/datastructs/circularbuffer.go b/src/pkg/circularbuffer/circularbuffer.go similarity index 84% rename from src/pkg/datastructs/circularbuffer.go rename to src/pkg/circularbuffer/circularbuffer.go index 5a24abd77..4ec990d5e 100644 --- a/src/pkg/datastructs/circularbuffer.go +++ b/src/pkg/circularbuffer/circularbuffer.go @@ -1,4 +1,4 @@ -package datastructs +package circularbuffer // BufferInterface abstracts the buffer operations type BufferInterface[T any] interface { @@ -21,7 +21,7 @@ func (c *CircularBuffer[T]) Add(item T) { func (c *CircularBuffer[T]) Get() []T { maxItems := min(c.entries, c.size) - items := make([]T, 0, maxItems) + items := make([]T, maxItems) startIdx := c.index // the c.index points to the next write position (ie. oldest entry) if the buffer is full, @@ -34,16 +34,16 @@ func (c *CircularBuffer[T]) Get() []T { // Collect items in chronological order for i := range maxItems { idx := (startIdx + i) % c.size - items = append(items, c.data[idx]) + items[i] = c.data[idx] } return items } -func NewCircularBuffer[T any](bufferSize int) CircularBuffer[T] { +func NewCircularBuffer[T any](bufferSize int) *CircularBuffer[T] { if bufferSize <= 0 { panic("failed to created a circular buffer: cannot have zero elements") } - return CircularBuffer[T]{ + return &CircularBuffer[T]{ size: bufferSize, entries: 0, index: 0, diff --git a/src/pkg/datastructs/circularbuffer_test.go b/src/pkg/circularbuffer/circularbuffer_test.go similarity index 95% rename from src/pkg/datastructs/circularbuffer_test.go rename to src/pkg/circularbuffer/circularbuffer_test.go index e2281d5a7..166a0775b 100644 --- a/src/pkg/datastructs/circularbuffer_test.go +++ b/src/pkg/circularbuffer/circularbuffer_test.go @@ -1,4 +1,4 @@ -package datastructs +package circularbuffer import ( "testing" diff --git a/src/pkg/cli/bootstrap.go b/src/pkg/cli/bootstrap.go index 50705d286..a2f5c7a05 100644 --- a/src/pkg/cli/bootstrap.go +++ b/src/pkg/cli/bootstrap.go @@ -9,7 +9,6 @@ import ( "github.com/DefangLabs/defang/src/pkg" "github.com/DefangLabs/defang/src/pkg/cli/client" - "github.com/DefangLabs/defang/src/pkg/datastructs" "github.com/DefangLabs/defang/src/pkg/dryrun" "github.com/DefangLabs/defang/src/pkg/logs" "github.com/DefangLabs/defang/src/pkg/term" @@ -51,10 +50,9 @@ func TailAndWaitForCD(ctx context.Context, provider client.Provider, projectName cancelTail(cdErr) }() - logCache := datastructs.NewCircularBuffer[string](30) // blocking call to tail var tailErr error - if err := streamLogs(ctx, provider, projectName, tailOptions, logCache, logEntryPrintHandler); err != nil { + if _, err := streamLogs(ctx, provider, projectName, tailOptions, logEntryPrintHandler); err != nil { term.Debug("Tail stopped with", err, errors.Unwrap(err)) if !errors.Is(err, context.Canceled) { tailErr = err diff --git a/src/pkg/cli/estimate.go b/src/pkg/cli/estimate.go index 67b2da98e..b9de7a0ae 100644 --- a/src/pkg/cli/estimate.go +++ b/src/pkg/cli/estimate.go @@ -13,7 +13,6 @@ import ( "github.com/DefangLabs/defang/src/pkg/cli/client" "github.com/DefangLabs/defang/src/pkg/cli/compose" - "github.com/DefangLabs/defang/src/pkg/datastructs" "github.com/DefangLabs/defang/src/pkg/logs" "github.com/DefangLabs/defang/src/pkg/modes" "github.com/DefangLabs/defang/src/pkg/money" @@ -77,8 +76,7 @@ func GeneratePreview(ctx context.Context, project *compose.Project, client clien Verbose: true, } - logCache := datastructs.NewCircularBuffer[string](30) - err = streamLogs(ctx, previewProvider, project.Name, tailOptions, logCache, func(entry *defangv1.LogEntry, options *TailOptions) error { + _, err = streamLogs(ctx, previewProvider, project.Name, tailOptions, func(entry *defangv1.LogEntry, options *TailOptions) error { if strings.HasPrefix(entry.Message, "Preview succeeded") { return io.EOF } else if strings.HasPrefix(entry.Message, "Preview failed") { diff --git a/src/pkg/cli/tail.go b/src/pkg/cli/tail.go index 89c60ed80..9b365a679 100644 --- a/src/pkg/cli/tail.go +++ b/src/pkg/cli/tail.go @@ -14,8 +14,8 @@ import ( "time" "github.com/DefangLabs/defang/src/pkg" + "github.com/DefangLabs/defang/src/pkg/circularbuffer" "github.com/DefangLabs/defang/src/pkg/cli/client" - "github.com/DefangLabs/defang/src/pkg/datastructs" "github.com/DefangLabs/defang/src/pkg/dryrun" "github.com/DefangLabs/defang/src/pkg/logs" "github.com/DefangLabs/defang/src/pkg/spinner" @@ -140,9 +140,7 @@ func (cerr CancelError) Unwrap() error { return cerr.error } -func Tail(ctx context.Context, provider client.Provider, projectName string, options TailOptions) (datastructs.BufferInterface[string], error) { - logCache := datastructs.NewCircularBuffer[string](30) - +func Tail(ctx context.Context, provider client.Provider, projectName string, options TailOptions) (circularbuffer.BufferInterface[string], error) { if options.LogType == logs.LogTypeUnspecified { options.LogType = logs.LogTypeAll } @@ -166,10 +164,10 @@ func Tail(ctx context.Context, provider client.Provider, projectName string, opt } if dryrun.DoDryRun { - return &logCache, dryrun.ErrDryRun + return nil, dryrun.ErrDryRun } - return &logCache, streamLogs(ctx, provider, projectName, options, logCache, logEntryPrintHandler) + return streamLogs(ctx, provider, projectName, options, logEntryPrintHandler) } func isTransientError(err error) bool { @@ -205,7 +203,8 @@ func isTransientError(err error) bool { type LogEntryHandler func(*defangv1.LogEntry, *TailOptions) error -func streamLogs(ctx context.Context, provider client.Provider, projectName string, options TailOptions, logCache datastructs.CircularBuffer[string], handler LogEntryHandler) error { +func streamLogs(ctx context.Context, provider client.Provider, projectName string, options TailOptions, handler LogEntryHandler) (circularbuffer.BufferInterface[string], error) { + logCache := circularbuffer.NewCircularBuffer[string](30) var sinceTs, untilTs *timestamppb.Timestamp if pkg.IsValidTime(options.Since) { sinceTs = timestamppb.New(options.Since) @@ -237,7 +236,7 @@ func streamLogs(ctx context.Context, provider client.Provider, projectName strin serverStream, err := provider.QueryLogs(ctx, tailRequest) if err != nil { - return err + return logCache, err } ctx, cancel := context.WithCancel(ctx) @@ -304,7 +303,7 @@ func streamLogs(ctx context.Context, provider client.Provider, projectName strin for { if !serverStream.Receive() { if errors.Is(serverStream.Err(), context.Canceled) || errors.Is(serverStream.Err(), context.DeadlineExceeded) { - return &CancelError{TailOptions: options, error: serverStream.Err(), ProjectName: projectName} + return logCache, &CancelError{TailOptions: options, error: serverStream.Err(), ProjectName: projectName} } // Reconnect on Error: internal: stream error: stream ID 5; INTERNAL_ERROR; received from peer @@ -315,13 +314,13 @@ func streamLogs(ctx context.Context, provider client.Provider, projectName strin spaces, _ = term.Warnf("Reconnecting...\r") // overwritten below } if err := provider.DelayBeforeRetry(ctx); err != nil { - return err + return logCache, err } tailRequest.Since = timestamppb.New(options.Since) serverStream, err = provider.QueryLogs(ctx, tailRequest) if err != nil { term.Debug("Reconnect failed:", err) - return err + return logCache, err } if !options.Raw { term.Printf("%*s", spaces, "\r") // clear the "reconnecting" message @@ -330,7 +329,7 @@ func streamLogs(ctx context.Context, provider client.Provider, projectName strin continue } - return serverStream.Err() // returns nil on EOF + return logCache, serverStream.Err() // returns nil on EOF } msg := serverStream.Msg() @@ -364,7 +363,7 @@ func streamLogs(ctx context.Context, provider client.Provider, projectName strin if options.EndEventDetectFunc != nil { if err := options.EndEventDetectFunc(e); err != nil { cancel() // TODO: stuck on defer Close() if we don't do this - return err + return logCache, err } } continue @@ -377,7 +376,7 @@ func streamLogs(ctx context.Context, provider client.Provider, projectName strin if err != nil { term.Debug("Ending tail loop", err) cancel() // TODO: stuck on defer Close() if we don't do this - return err + return logCache, err } } } diff --git a/src/pkg/cli/tailAndMonitor.go b/src/pkg/cli/tailAndMonitor.go index 5292a0334..77d8e6a69 100644 --- a/src/pkg/cli/tailAndMonitor.go +++ b/src/pkg/cli/tailAndMonitor.go @@ -8,9 +8,9 @@ import ( "time" "github.com/DefangLabs/defang/src/pkg" + "github.com/DefangLabs/defang/src/pkg/circularbuffer" "github.com/DefangLabs/defang/src/pkg/cli/client" "github.com/DefangLabs/defang/src/pkg/cli/compose" - "github.com/DefangLabs/defang/src/pkg/datastructs" "github.com/DefangLabs/defang/src/pkg/term" defangv1 "github.com/DefangLabs/defang/src/protos/io/defang/v1" "github.com/bufbuild/connect-go" @@ -18,7 +18,7 @@ import ( const targetServiceState = defangv1.ServiceState_DEPLOYMENT_COMPLETED -func TailAndMonitor(ctx context.Context, project *compose.Project, provider client.Provider, waitTimeout time.Duration, tailOptions TailOptions) (ServiceStates, datastructs.BufferInterface[string], error) { +func TailAndMonitor(ctx context.Context, project *compose.Project, provider client.Provider, waitTimeout time.Duration, tailOptions TailOptions) (ServiceStates, circularbuffer.BufferInterface[string], error) { if tailOptions.Deployment == "" { panic("tailOptions.Deployment must be a valid deployment ID") } @@ -68,7 +68,7 @@ func TailAndMonitor(ctx context.Context, project *compose.Project, provider clie // blocking call to tail var err, tailErr error - var logCache datastructs.BufferInterface[string] + var logCache circularbuffer.BufferInterface[string] if logCache, err = Tail(tailCtx, provider, project.Name, tailOptions); err != nil { term.Debug("Tail stopped with", err, errors.Unwrap(err)) diff --git a/src/pkg/mcp/tools/default_tool_cli.go b/src/pkg/mcp/tools/default_tool_cli.go index 999849ffb..6057dbe30 100644 --- a/src/pkg/mcp/tools/default_tool_cli.go +++ b/src/pkg/mcp/tools/default_tool_cli.go @@ -7,10 +7,10 @@ import ( "os" "strconv" + "github.com/DefangLabs/defang/src/pkg/circularbuffer" "github.com/DefangLabs/defang/src/pkg/cli" cliClient "github.com/DefangLabs/defang/src/pkg/cli/client" "github.com/DefangLabs/defang/src/pkg/cli/compose" - "github.com/DefangLabs/defang/src/pkg/datastructs" "github.com/DefangLabs/defang/src/pkg/login" "github.com/DefangLabs/defang/src/pkg/mcp/common" "github.com/DefangLabs/defang/src/pkg/mcp/deployment_info" @@ -57,7 +57,7 @@ func (DefaultToolCLI) ComposeUp(ctx context.Context, project *compose.Project, c return cli.ComposeUp(ctx, project, client, provider, uploadMode, mode) } -func (DefaultToolCLI) Tail(ctx context.Context, provider cliClient.Provider, project *compose.Project, options cli.TailOptions) (datastructs.BufferInterface[string], error) { +func (DefaultToolCLI) Tail(ctx context.Context, provider cliClient.Provider, project *compose.Project, options cli.TailOptions) (circularbuffer.BufferInterface[string], error) { return cli.Tail(ctx, provider, project.Name, options) } diff --git a/src/pkg/mcp/tools/logs.go b/src/pkg/mcp/tools/logs.go index ffcc09196..4959f4e09 100644 --- a/src/pkg/mcp/tools/logs.go +++ b/src/pkg/mcp/tools/logs.go @@ -5,10 +5,10 @@ import ( "fmt" "time" + "github.com/DefangLabs/defang/src/pkg/circularbuffer" cliTypes "github.com/DefangLabs/defang/src/pkg/cli" cliClient "github.com/DefangLabs/defang/src/pkg/cli/client" "github.com/DefangLabs/defang/src/pkg/cli/compose" - "github.com/DefangLabs/defang/src/pkg/datastructs" "github.com/DefangLabs/defang/src/pkg/term" "github.com/mark3labs/mcp-go/mcp" ) @@ -53,7 +53,7 @@ type LogsCLIInterface interface { connecter providerFactory // Unique methods - Tail(ctx context.Context, provider cliClient.Provider, project *compose.Project, options cliTypes.TailOptions) (datastructs.BufferInterface[string], error) + Tail(ctx context.Context, provider cliClient.Provider, project *compose.Project, options cliTypes.TailOptions) (circularbuffer.BufferInterface[string], error) CheckProviderConfigured(ctx context.Context, client *cliClient.GrpcClient, providerId cliClient.ProviderID, projectName string, serviceCount int) (cliClient.Provider, error) LoadProject(ctx context.Context, loader cliClient.Loader) (*compose.Project, error) } diff --git a/src/pkg/track/track.go b/src/pkg/track/track.go index 115e72837..b4a3ea4a8 100644 --- a/src/pkg/track/track.go +++ b/src/pkg/track/track.go @@ -15,7 +15,8 @@ import ( const maxPropertyCharacterLength = 255 // chars per property in tracking event var disableAnalytics = pkg.GetenvBool("DEFANG_DISABLE_ANALYTICS") -var logPropertyNamePrefix = "logs" + +const logPropertyNamePrefix = "logs" type Property = cliClient.Property @@ -48,7 +49,7 @@ func Evt(name string, props ...Property) { return } - // filter out props with a name prefix of logPropertyNamePrefix, they should already be in the debug output + // compose logs may be in the tracking, they can be large so filter them out from debug output var filteredProps []Property for _, p := range props { if strings.HasPrefix(p.Name, logPropertyNamePrefix) {