diff --git a/.dockerignore b/.dockerignore index 6b1a85acb8..7bddf77b7c 100644 --- a/.dockerignore +++ b/.dockerignore @@ -7,3 +7,4 @@ internal/web/ui/build/ packaging/windows/LICENSE packaging/windows/agent-windows-amd64.exe cmd/grafana-agent/Dockerfile +alloy diff --git a/.github/workflows/test_pyroscope_pr.yml b/.github/workflows/test_pyroscope_pr.yml index 46ee062364..8606937e3e 100644 --- a/.github/workflows/test_pyroscope_pr.yml +++ b/.github/workflows/test_pyroscope_pr.yml @@ -26,4 +26,4 @@ jobs: go-version-file: go.mod cache: false - - run: make GO_TAGS="nodocker" test-pyroscope \ No newline at end of file + - run: sudo make test-pyroscope diff --git a/.gitignore b/.gitignore index 9ddcfe6ade..905fa90beb 100644 --- a/.gitignore +++ b/.gitignore @@ -33,3 +33,4 @@ node_modules # file of the pair will detect a dirty work tree and detect the wrong tag name. .tag-only .image-tag +alloy diff --git a/internal/component/pyroscope/java/integration/integration_test.go b/internal/component/pyroscope/java/integration/integration_test.go new file mode 100644 index 0000000000..e7186b403d --- /dev/null +++ b/internal/component/pyroscope/java/integration/integration_test.go @@ -0,0 +1,137 @@ +//go:build linux && (amd64 || arm64) + +package integration + +import ( + "context" + "fmt" + "net/http" + "os" + "strings" + "sync" + "testing" + "time" + + "github.com/go-kit/log" + "github.com/grafana/alloy/internal/component/discovery" + "github.com/grafana/alloy/internal/component/pyroscope" + "github.com/grafana/alloy/internal/component/pyroscope/java" + "github.com/grafana/alloy/internal/component/pyroscope/testutil" + "github.com/grafana/alloy/internal/component/pyroscope/util/test" + pyroutil "github.com/grafana/alloy/internal/component/pyroscope/util/test/container" + querierv1 "github.com/grafana/pyroscope/api/gen/proto/go/querier/v1" + "github.com/prometheus/client_golang/prometheus" + "github.com/stretchr/testify/require" +) + +func TestPyroscopeJavaIntegration(t *testing.T) { + if os.Getenv("GITHUB_ACTIONS") == "true" && os.Getenv("GITHUB_JOB") != "test_pyroscope" { + t.Skip("Skipping Pyroscope Java integration test in GitHub Actions (job name is not test_pyroscope)") + } + wg := sync.WaitGroup{} + defer func() { + wg.Wait() + }() + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + l := log.NewSyncLogger(log.NewLogfmtLogger(os.Stderr)) + l = log.WithPrefix(l, + "test", t.Name(), + "ts", log.DefaultTimestampUTC, + ) + + _, pyroscopeEndpoint := pyroutil.StartPyroscopeContainer(t, ctx, l) + + _, javaEndpoint, pid := pyroutil.StartJavaApplicationContainer(t, ctx, l) + + t.Logf("Pyroscope endpoint: %s", pyroscopeEndpoint) + t.Logf("Java application endpoint: %s", javaEndpoint) + t.Logf("Java process PID in container: %d", pid) + + reg := prometheus.NewRegistry() + + writeComponent, err := testutil.CreateWriteComponent(l, reg, pyroscopeEndpoint) + require.NoError(t, err, "Failed to create write component") + + args := java.DefaultArguments() + args.ForwardTo = []pyroscope.Appendable{writeComponent} + args.ProfilingConfig.Interval = time.Second + args.Targets = []discovery.Target{ + discovery.NewTargetFromMap(map[string]string{ + java.LabelProcessID: fmt.Sprintf("%d", pid), + "service_name": "spring-petclinic", + }), + } + javaComponent, err := java.New( + log.With(l, "component", "pyroscope.java"), + reg, + "test-java", + args, + ) + require.NoError(t, err, "Failed to create java component") + + wg.Add(2) + go func() { + defer wg.Done() + _ = javaComponent.Run(ctx) + }() + go func() { + defer wg.Done() + for ctx.Err() == nil { + burn(javaEndpoint) + time.Sleep(100 * time.Millisecond) + } + }() + + require.Eventually(t, func() bool { + req := &querierv1.SelectMergeProfileRequest{ + ProfileTypeID: `process_cpu:cpu:nanoseconds:cpu:nanoseconds`, + LabelSelector: `{service_name="spring-petclinic"}`, + Start: time.Now().Add(-time.Hour).UnixMilli(), + End: time.Now().UnixMilli(), + } + res, err := test.Query(pyroscopeEndpoint, req) + if err != nil { + t.Logf("Error querying endpoint: %v", err) + return false + } + ss := res.String() + if !strings.Contains(ss, `org/springframework/samples/petclinic/web/VetController.showVetList`) { + return false + } + if !strings.Contains(ss, `libjvm.so.JavaThread::thread_main_inner`) { + return false + } + return true + }, 90*time.Second, 100*time.Millisecond) + + require.Eventually(t, func() bool { + req := &querierv1.SelectMergeProfileRequest{ + ProfileTypeID: `memory:alloc_in_new_tlab_bytes:bytes:space:bytes`, + LabelSelector: `{service_name="spring-petclinic"}`, + Start: time.Now().Add(-time.Hour).UnixMilli(), + End: time.Now().UnixMilli(), + } + res, err := test.Query(pyroscopeEndpoint, req) + if err != nil { + t.Logf("Error querying endpoint: %v", err) + return false + } + ss := res.String() + if !strings.Contains(ss, `org/springframework/samples/petclinic/web/VetController.showVetList`) { + return false + } + if strings.Contains(ss, `libjvm.so.JavaThread::thread_main_inner`) { + return false + } + return true + }, 90*time.Second, 100*time.Millisecond) + cancel() +} + +func burn(url string) { + _, _ = http.DefaultClient.Get(url + "/") + _, _ = http.DefaultClient.Get(url + "/owners/find") + _, _ = http.DefaultClient.Get(url + "/vets") + _, _ = http.DefaultClient.Get(url + "/oups") +} diff --git a/internal/component/pyroscope/testutil/components.go b/internal/component/pyroscope/testutil/components.go new file mode 100644 index 0000000000..fbf0ef75eb --- /dev/null +++ b/internal/component/pyroscope/testutil/components.go @@ -0,0 +1,36 @@ +//go:build linux && (arm64 || amd64) + +package testutil + +import ( + "fmt" + + "github.com/go-kit/log" + "github.com/grafana/alloy/internal/component/pyroscope" + "github.com/grafana/alloy/internal/component/pyroscope/write" + "github.com/prometheus/client_golang/prometheus" + "go.opentelemetry.io/otel/trace/noop" +) + +// CreateWriteComponent creates a pyroscope.write component that forwards to the given endpoint +func CreateWriteComponent(l log.Logger, reg prometheus.Registerer, endpoint string) (pyroscope.Appendable, error) { + var receiver pyroscope.Appendable + e := write.GetDefaultEndpointOptions() + e.URL = endpoint + + _, err := write.New( + log.With(l, "component", "pyroscope.write"), + noop.Tracer{}, + reg, + func(exports write.Exports) { + receiver = exports.Receiver + }, + "test", + "", + write.Arguments{Endpoints: []*write.EndpointOptions{&e}}, + ) + if err != nil { + return nil, fmt.Errorf("error creating write component: %w", err) + } + return receiver, nil +} diff --git a/internal/component/pyroscope/util/internal/cmd/playground/main.go b/internal/component/pyroscope/util/internal/cmd/playground/main.go index 45e1d293dd..400c88d351 100644 --- a/internal/component/pyroscope/util/internal/cmd/playground/main.go +++ b/internal/component/pyroscope/util/internal/cmd/playground/main.go @@ -14,10 +14,9 @@ import ( "github.com/grafana/alloy/internal/component/pyroscope" "github.com/grafana/alloy/internal/component/pyroscope/ebpf" "github.com/grafana/alloy/internal/component/pyroscope/java" - "github.com/grafana/alloy/internal/component/pyroscope/write" + "github.com/grafana/alloy/internal/component/pyroscope/testutil" "github.com/oklog/run" "github.com/prometheus/client_golang/prometheus" - "go.opentelemetry.io/otel/trace/noop" ) var ( @@ -39,20 +38,7 @@ func parseConfig() *config { } func newWrite() pyroscope.Appendable { - var receiver pyroscope.Appendable - e := write.GetDefaultEndpointOptions() - e.URL = "http://localhost:4040" - _, err := write.New( - log.With(l, "component", "write"), - noop.Tracer{}, - reg, - func(exports write.Exports) { - receiver = exports.Receiver - }, - "playground", - "", - write.Arguments{Endpoints: []*write.EndpointOptions{&e}}, - ) + receiver, err := testutil.CreateWriteComponent(l, reg, "http://localhost:4040") if err != nil { _ = l.Log("msg", "error creating write component", "err", err) os.Exit(1) diff --git a/internal/component/pyroscope/util/test/container/java.go b/internal/component/pyroscope/util/test/container/java.go new file mode 100644 index 0000000000..2942dd0eef --- /dev/null +++ b/internal/component/pyroscope/util/test/container/java.go @@ -0,0 +1,54 @@ +package container + +import ( + "context" + "fmt" + stdlog "log" + "testing" + "time" + + "github.com/docker/docker/api/types/container" + "github.com/docker/go-connections/nat" + "github.com/go-kit/log" + "github.com/stretchr/testify/require" + "github.com/testcontainers/testcontainers-go" + "github.com/testcontainers/testcontainers-go/wait" +) + +func StartJavaApplicationContainer(t *testing.T, ctx context.Context, l log.Logger) (testcontainers.Container, string, int) { + req := testcontainers.ContainerRequest{ + Image: "springcommunity/spring-framework-petclinic:latest", + ExposedPorts: []string{"8080/tcp"}, + WaitingFor: wait.ForHTTP("/").WithPort("8080/tcp").WithStartupTimeout(3 * time.Minute), + Env: map[string]string{ + "JAVA_OPTS": "-Xmx512m -Xms256m", + }, + HostConfigModifier: func(hc *container.HostConfig) { + hc.PidMode = "host" + }, + } + + c, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ + ContainerRequest: req, + Started: true, + Logger: stdlog.New(log.NewStdlibAdapter(l), "", 0), + }) + require.NoError(t, err) + + t.Cleanup(func() { + err := testcontainers.TerminateContainer(c) + require.NoError(t, err) + }) + + mappedPort, err := c.MappedPort(ctx, nat.Port("8080/tcp")) + require.NoError(t, err) + + host, err := c.Host(ctx) + require.NoError(t, err) + + endpoint := fmt.Sprintf("http://%s:%s", host, mappedPort.Port()) + inspected, err := c.Inspect(t.Context()) + require.NoError(t, err) + + return c, endpoint, inspected.State.Pid +} diff --git a/internal/component/pyroscope/util/test/container/pyroscope.go b/internal/component/pyroscope/util/test/container/pyroscope.go new file mode 100644 index 0000000000..cfde5ae261 --- /dev/null +++ b/internal/component/pyroscope/util/test/container/pyroscope.go @@ -0,0 +1,46 @@ +package container + +import ( + "context" + "fmt" + stdlog "log" + "testing" + + "github.com/go-kit/log" + "github.com/stretchr/testify/require" + "github.com/testcontainers/testcontainers-go" + "github.com/testcontainers/testcontainers-go/wait" +) + +func StartPyroscopeContainer(t *testing.T, ctx context.Context, l log.Logger) (testcontainers.Container, string) { + req := testcontainers.ContainerRequest{ + Image: "grafana/pyroscope:latest", + Cmd: []string{"--ingester.min-ready-duration=0s"}, + ExposedPorts: []string{"4040/tcp"}, + WaitingFor: wait.ForHTTP("/ready").WithPort("4040/tcp"), + Env: map[string]string{ + "PYROSCOPE_LOG_LEVEL": "debug", + }, + } + + c, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ + ContainerRequest: req, + Started: true, + Logger: stdlog.New(log.NewStdlibAdapter(l), "", 0), + }) + require.NoError(t, err) + + t.Cleanup(func() { + err := testcontainers.TerminateContainer(c) + require.NoError(t, err) + }) + + mappedPort, err := c.MappedPort(ctx, "4040/tcp") + require.NoError(t, err) + + host, err := c.Host(ctx) + require.NoError(t, err) + + endpoint := fmt.Sprintf("http://%s:%s", host, mappedPort.Port()) + return c, endpoint +} diff --git a/internal/component/pyroscope/util/test/query.go b/internal/component/pyroscope/util/test/query.go new file mode 100644 index 0000000000..9cb8ed3e3c --- /dev/null +++ b/internal/component/pyroscope/util/test/query.go @@ -0,0 +1,24 @@ +package test + +import ( + "context" + "net/http" + + "connectrpc.com/connect" + "github.com/google/pprof/profile" + querierv1 "github.com/grafana/pyroscope/api/gen/proto/go/querier/v1" + "github.com/grafana/pyroscope/api/gen/proto/go/querier/v1/querierv1connect" +) + +func Query(url string, q *querierv1.SelectMergeProfileRequest) (*profile.Profile, error) { + client := querierv1connect.NewQuerierServiceClient(http.DefaultClient, url) + res, err := client.SelectMergeProfile(context.Background(), connect.NewRequest(q)) + if err != nil { + return nil, err + } + bs, err := res.Msg.MarshalVT() + if err != nil { + return nil, err + } + return profile.ParseData(bs) +}