Skip to content

Commit be62060

Browse files
atherielatheriel
committed
Add basic Open Telemetry instrumentation for all requests.
This commit wraps all requests in an Open Telemetry span that abides by the semantic conventions for HTTP clients [0] (insofar as I understand them). We also propagate the trace context [1] when there is one. Right now this instrumentation is opt in: `otel` is in `Suggests`, and tracing must be enabled (e.g. via the `OTEL_TRACES_EXPORTER` environment variable). Otherwise this is costless at runtime. For example: library(otelsdk) Sys.setenv(OTEL_TRACES_EXPORTER = "stderr") request("https://google.com") |> req_perform() I'm not sure that `otel` needs to move to `Imports`, because by design users actually need the `otelsdk` package to enable tracing anyway. Unit tests are included. [0]: https://opentelemetry.io/docs/specs/semconv/http/http-spans/#http-client-span [1]: https://www.w3.org/TR/trace-context/ Signed-off-by: Aaron Jacobs <[email protected]>
1 parent df68c2b commit be62060

File tree

6 files changed

+235
-7
lines changed

6 files changed

+235
-7
lines changed

‎DESCRIPTION‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@ Suggests:
3939
knitr,
4040
later (>= 1.4.0),
4141
nanonext,
42+
otel (>= 0.0.0.9000),
4243
paws.common,
4344
promises,
4445
rmarkdown,
@@ -56,4 +57,5 @@ Encoding: UTF-8
5657
Roxygen: list(markdown = TRUE)
5758
RoxygenNote: 7.3.2
5859
Remotes:
59-
r-lib/webfakes
60+
r-lib/webfakes,
61+
r-lib/otel

‎NEWS.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
# httr2 (development version)
22

33
* `req_url_query()` now re-calculates n lengths when using `.multi = "explode"` to avoid select/recycling issues (@Kevanness, #719).
4+
* httr2 will now emit OpenTelemetry traces for all requests when tracing is enabled. Requires the `otelsdk` package (@atheriel, #729).
45

56
# httr2 1.1.2
67

‎R/otel.R‎

Lines changed: 116 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,116 @@
1+
# Attaches an Open Telemetry span that abides by the semantic conventions for
2+
# HTTP clients to the request, including the associated W3C trace context
3+
# headers.
4+
#
5+
# See: https://opentelemetry.io/docs/specs/semconv/http/http-spans/#http-client-span
6+
req_with_span <- function(
7+
req,
8+
resend_count = 0,
9+
tracer = default_tracer(),
10+
scope = parent.frame()
11+
) {
12+
if (is.null(tracer) || !tracer$is_enabled()) {
13+
return(req)
14+
}
15+
parsed <- tryCatch(url_parse(req$url), error = function(cnd) NULL)
16+
if (is.null(parsed)) {
17+
# Don't create spans for invalid URLs.
18+
return(req)
19+
}
20+
if (!req_has_user_agent(req)) {
21+
req <- req_user_agent(req)
22+
}
23+
default_port <- 443L
24+
if (parsed$scheme == "http") {
25+
default_port <- 80L
26+
}
27+
# Follow the semantic conventions and redact credentials in the URL, when
28+
# present.
29+
if (!is.null(parsed$username)) {
30+
parsed$username <- "REDACTED"
31+
}
32+
if (!is.null(parsed$password)) {
33+
parsed$password <- "REDACTED"
34+
}
35+
method <- req_method_get(req)
36+
span <- tracer$start_span(
37+
name = method,
38+
options = list(kind = "CLIENT"),
39+
# Ensure we set attributes relevant to sampling at span creation time.
40+
attributes = compact(list(
41+
"http.request.method" = method,
42+
"server.address" = parsed$hostname,
43+
"server.port" = parsed$port %||% default_port,
44+
"url.full" = url_build(parsed),
45+
"http.request.resend_count" = if (resend_count > 1) resend_count,
46+
"user_agent.original" = req$options$useragent
47+
)),
48+
scope = scope
49+
)
50+
ctx <- span$get_context()
51+
req <- req_headers(req, !!!ctx$to_http_headers())
52+
req$state$span <- span
53+
req
54+
}
55+
56+
# Ends the Open Telemetry span associated with this request, if any.
57+
req_end_span <- function(req, resp = NULL) {
58+
span <- req$state$span
59+
if (is.null(span) || !span$is_recording()) {
60+
return()
61+
}
62+
if (is.null(resp)) {
63+
span$end()
64+
return()
65+
}
66+
if (is_error(resp)) {
67+
span$record_exception(resp)
68+
span$set_status("error")
69+
# Surface the underlying curl error class.
70+
span$set_attribute("error.type", class(resp$parent)[1])
71+
span$end()
72+
return()
73+
}
74+
span$set_attribute("http.response.status_code", resp_status(resp))
75+
if (error_is_error(req, resp)) {
76+
desc <- resp_status_desc(resp)
77+
if (is.na(desc)) {
78+
desc <- NULL
79+
}
80+
span$set_status("error", desc)
81+
# The semantic conventions recommend using the status code as a string for
82+
# these cases.
83+
span$set_attribute("error.type", as.character(resp_status(resp)))
84+
} else {
85+
span$set_status("ok")
86+
}
87+
span$end()
88+
}
89+
90+
# Replaces the existing Open Telemetry span on a request with a new one. Used
91+
# for retries.
92+
req_reset_span <- function(
93+
req,
94+
handle,
95+
resend_count = 0,
96+
tracer = default_tracer(),
97+
scope = parent.frame()
98+
) {
99+
req <- req_with_span(req, resend_count, tracer, scope)
100+
if (is.null(req$state$span)) {
101+
return(req)
102+
}
103+
# Because the headers have changed, we need to re-sign the request and update
104+
# stateful components (like the handle).
105+
req <- auth_sign(req)
106+
curl::handle_setheaders(handle, .list = headers_flatten(req$headers))
107+
req$state$headers <- req$headers
108+
req
109+
}
110+
111+
default_tracer <- function() {
112+
if (!is_installed("otel")) {
113+
return(NULL)
114+
}
115+
otel::get_tracer("httr2")
116+
}

‎R/req-perform-connection.R‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,7 @@ req_perform_connection <- function(req, blocking = TRUE, verbosity = NULL) {
5454
# verbosity checked in req_verbosity_connection
5555

5656
req <- req_verbosity_connection(req, verbosity %||% httr2_verbosity())
57+
req <- req_with_span(req)
5758
req_prep <- req_prepare(req)
5859
handle <- req_handle(req_prep)
5960
the$last_request <- req
@@ -71,7 +72,14 @@ req_perform_connection <- function(req, blocking = TRUE, verbosity = NULL) {
7172
if (!is.null(resp)) {
7273
close(resp)
7374
}
75+
76+
if (tries != 0) {
77+
# Start a new span for retried requests.
78+
req_prep <- req_reset_span(req_prep, handle, resend_count = tries)
79+
}
80+
7481
resp <- req_perform_connection1(req, handle, blocking = blocking)
82+
req_completed(req_prep, resp)
7583

7684
if (retry_is_transient(req, resp)) {
7785
tries <- tries + 1
@@ -82,7 +90,6 @@ req_perform_connection <- function(req, blocking = TRUE, verbosity = NULL) {
8290
break
8391
}
8492
}
85-
req_completed(req)
8693

8794
if (!is_error(resp) && error_is_error(req, resp)) {
8895
# Read full body if there's an error

‎R/req-perform.R‎

Lines changed: 13 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -79,6 +79,8 @@ req_perform <- function(
7979
verbosity <- verbosity %||% httr2_verbosity()
8080

8181
if (!is.null(mock)) {
82+
# Allow us to use with_mock() to confirm context propagation.
83+
req <- req_with_span(req)
8284
mock <- as_function(mock)
8385
mock_resp <- mock(req)
8486
if (!is.null(mock_resp)) {
@@ -93,6 +95,9 @@ req_perform <- function(
9395
return(req)
9496
}
9597

98+
sys_sleep(throttle_delay(req), "for throttling delay")
99+
100+
req <- req_with_span(req)
96101
req_prep <- req_prepare(req)
97102
handle <- req_handle(req_prep)
98103
max_tries <- retry_max_tries(req)
@@ -101,17 +106,19 @@ req_perform <- function(
101106
n <- 0
102107
tries <- 0
103108
reauthed <- FALSE # only ever re-authenticate once
104-
105-
sys_sleep(throttle_delay(req), "for throttling delay")
106-
107109
delay <- 0
108110
while (tries < max_tries && Sys.time() < deadline) {
109111
retry_check_breaker(req, tries, error_call = error_call)
110112
sys_sleep(delay, "for retry backoff")
111113
n <- n + 1
112114

115+
if (n != 1) {
116+
# Start a new span for retried requests.
117+
req_prep <- req_reset_span(req_prep, handle, resend_count = n)
118+
}
119+
113120
resp <- req_perform1(req, path = path, handle = handle)
114-
req_completed(req_prep)
121+
req_completed(req_prep, resp)
115122

116123
if (retry_is_transient(req, resp)) {
117124
tries <- tries + 1
@@ -269,7 +276,8 @@ req_handle <- function(req) {
269276

270277
handle
271278
}
272-
req_completed <- function(req) {
279+
req_completed <- function(req, resp = NULL) {
280+
req_end_span(req, resp)
273281
req_policy_call(req, "done", list(), NULL)
274282
}
275283

‎tests/testthat/test-req-perform.R‎

Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -216,3 +216,97 @@ test_that("checks input types", {
216216
req_perform(req, mock = 7)
217217
})
218218
})
219+
220+
test_that("tracing works as expected", {
221+
skip_if_not_installed("otelsdk")
222+
223+
spans <- otelsdk::with_otel_record({
224+
# A request with no URL (which shouldn't create a span).
225+
try(req_perform(request("")), silent = TRUE)
226+
227+
# A regular request.
228+
req_perform(request_test())
229+
230+
# A request with an HTTP error.
231+
try(
232+
req_perform(request_test("/status/:status", status = 404)),
233+
silent = TRUE
234+
)
235+
236+
# A request with basic credentials that we should redact.
237+
with_mocked_responses(
238+
function(req) {
239+
# Verify that the traceparent header is present when tracing while
240+
# we're in here.
241+
expect_false(is.null(req$headers$traceparent))
242+
response(status_code = 200)
243+
},
244+
req_perform(request("https://test:[email protected]"))
245+
)
246+
247+
# A request with a curl error.
248+
with_mocked_bindings(
249+
try(req_perform(request("http://127.0.0.1")), silent = TRUE),
250+
curl_fetch = function(...) abort("Failed to connect")
251+
)
252+
253+
# A request that triggers retries, generating three individual spans.
254+
request_test("/status/:status", status = 429) %>%
255+
req_retry(max_tries = 3, backoff = ~0) %>%
256+
req_perform() %>%
257+
try(silent = TRUE)
258+
})[["traces"]]
259+
260+
expect_length(spans, 7L)
261+
262+
# Validate the span for regular requests.
263+
expect_equal(spans[[1]]$status, "ok")
264+
expect_named(
265+
spans[[1]]$attributes,
266+
c(
267+
"http.response.status_code",
268+
"user_agent.original",
269+
"url.full",
270+
"server.address",
271+
"server.port",
272+
"http.request.method"
273+
)
274+
)
275+
expect_equal(spans[[1]]$attributes$http.request.method, "GET")
276+
expect_equal(spans[[1]]$attributes$http.response.status_code, 200L)
277+
expect_equal(spans[[1]]$attributes$server.address, "127.0.0.1")
278+
expect_match(spans[[1]]$attributes$user_agent.original, "^httr2/")
279+
280+
# And for requests with HTTP errors.
281+
expect_equal(spans[[2]]$status, "error")
282+
expect_equal(spans[[2]]$description, "Not Found")
283+
expect_equal(spans[[2]]$attributes$http.response.status_code, 404L)
284+
expect_equal(spans[[2]]$attributes$error.type, "404")
285+
286+
# And for spans with redacted credentials.
287+
expect_equal(spans[[3]]$attributes$server.address, "example.com")
288+
expect_equal(spans[[3]]$attributes$server.port, 443L)
289+
expect_equal(
290+
spans[[3]]$attributes$url.full,
291+
"https://REDACTED:[email protected]/"
292+
)
293+
294+
# And for spans with curl errors.
295+
expect_equal(spans[[4]]$status, "error")
296+
expect_equal(spans[[4]]$attributes$error.type, "rlang_error")
297+
298+
# We should have attached the curl error as an event.
299+
expect_length(spans[[4]]$events, 1L)
300+
expect_equal(spans[[4]]$events[[1]]$name, "exception")
301+
302+
# For spans with retries, we expect the parent context to be the same for
303+
# each span. (In this case, there is no parent span, so it should be empty.)
304+
# It is important that they not be children of one another.
305+
expect_equal(spans[[5]]$parent, "0000000000000000")
306+
expect_equal(spans[[6]]$parent, "0000000000000000")
307+
expect_equal(spans[[7]]$parent, "0000000000000000")
308+
309+
# Verify that we set resend counts correctly.
310+
expect_equal(spans[[6]]$attributes$http.request.resend_count, 2L)
311+
expect_equal(spans[[7]]$attributes$http.request.resend_count, 3L)
312+
})

0 commit comments

Comments
 (0)