http2: send graceful goaways on close

api/envoy/config/core/v3/protocol.proto

@@ -473,7 +473,7 @@ message KeepaliveSettings {
       [(validate.rules).duration = {gte {nanos: 1000000}}];
 }
 
-// [#next-free-field: 18]
+// [#next-free-field: 19]
 message Http2ProtocolOptions {
   option (udpa.annotations.versioning).previous_message_type =
       "envoy.api.v2.core.Http2ProtocolOptions";
@@ -662,6 +662,14 @@ message Http2ProtocolOptions {
 
   // Configure the maximum amount of metadata than can be handled per stream. Defaults to 1 MB.
   google.protobuf.UInt64Value max_metadata_size = 17;
+
+  // Timeout for graceful HTTP/2 GOAWAY shutdown sequence.
+  // When graceful shutdown is enabled, Envoy will send an initial GOAWAY frame with
+  // last_stream_id set to 2^31-1, wait for this timeout period, then send a final
+  // GOAWAY frame with the actual highest received stream ID. This allows in-flight
+  // requests to complete gracefully. If set to 0, graceful shutdown is disabled
+  // and the standard immediate GOAWAY behavior is used. Defaults to 1000ms (1 second).
+  google.protobuf.Duration graceful_goaway_timeout = 18;
 }
 
 // [#not-implemented-hide:]

changelogs/current.yaml

@@ -548,5 +548,14 @@ new_features:
   change: |
     Added ``setUpstreamOverrideHost`` method to AsyncClient StreamOptions to enable direct host routing
     that bypasses load balancer selection.
+- area: http2
+  change: |
+    Enabled graceful HTTP/2 GOAWAY shutdown by default with a 1-second timeout. The
+    :ref:`graceful_goaway_timeout <envoy_v3_api_field_config.core.v3.Http2ProtocolOptions.graceful_goaway_timeout>`
+    field now defaults to 1000ms (1 second) instead of 0 (disabled). This allows in-flight
+    requests to complete gracefully during connection shutdown by sending an initial GOAWAY
+    with max stream ID, waiting for the timeout, then sending a final GOAWAY. Set to 0 to
+    disable graceful shutdown. This feature can be disabled using the runtime guard
+    ``envoy.reloadable_features.http2_graceful_goaway``.
 
 deprecated:

envoy/http/codec.h

@@ -586,6 +586,12 @@ class Connection {
    */
   virtual void goAway() PURE;
 
+  /**
+   * Indicate graceful "go away" to the remote. For HTTP/2, this implements RFC-compliant
+   * graceful shutdown. For other protocols, this falls back to regular goAway().
+   */
+  virtual void goAwayGraceful() { goAway(); }
+
   /**
    * @return the protocol backing the connection. This can change if for example an HTTP/1.1
    *         connection gets an HTTP/1.0 request on it.

source/common/http/codec_client.h

@@ -89,6 +89,12 @@ class CodecClient : protected Logger::Loggable<Logger::Id::client>,
    */
   void goAway() { codec_->goAway(); }
 
+  /**
+   * Send a graceful codec level go away indication to the peer.
+   * For HTTP/2, this implements RFC-compliant graceful shutdown.
+   */
+  void goAwayGraceful() { codec_->goAwayGraceful(); }
+
   /**
    * @return the underlying connection ID.
    */

source/common/http/conn_manager_impl.cc

@@ -786,7 +786,15 @@ void ConnectionManagerImpl::sendGoAwayAndClose() {
   if (go_away_sent_) {
     return;
   }
-  codec_->goAway();
+
+  // Try graceful GOAWAY for HTTP/2 connections, fallback to regular GOAWAY
+  auto* http2_conn = dynamic_cast<Http2::ConnectionImpl*>(codec_.get());
+  if (http2_conn != nullptr) {
+    http2_conn->goAwayGraceful();
+  } else {
+    codec_->goAway();
+  }
+
   go_away_sent_ = true;
   doConnectionClose(Network::ConnectionCloseType::FlushWriteAndDelay, absl::nullopt,
                     "forced_goaway");

source/common/http/http2/codec_impl.cc

@@ -22,6 +22,7 @@
 #include "source/common/http/headers.h"
 #include "source/common/http/http2/codec_stats.h"
 #include "source/common/http/utility.h"
+#include "source/common/protobuf/utility.h"
 #include "source/common/runtime/runtime_features.h"
 
 #include "absl/cleanup/cleanup.h"
@@ -865,7 +866,7 @@ ConnectionImpl::ConnectionImpl(Network::Connection& connection, CodecStats& stat
       stream_error_on_invalid_http_messaging_(
           http2_options.override_stream_error_on_invalid_http_message().value()),
       protocol_constraints_(stats, http2_options), dispatching_(false), raised_goaway_(false),
-      random_(random_generator),
+      graceful_goaway_in_progress_(false), random_(random_generator),
       last_received_data_time_(connection_.dispatcher().timeSource().monotonicTime()) {
   if (http2_options.has_use_oghttp2_codec()) {
     use_oghttp2_library_ = http2_options.use_oghttp2_codec().value();
@@ -890,6 +891,14 @@ ConnectionImpl::ConnectionImpl(Network::Connection& connection, CodecStats& stat
     // This call schedules the initial interval, with jitter.
     onKeepaliveResponse();
   }
+
+  // Initialize graceful GOAWAY timeout
+  graceful_goaway_timeout_ = std::chrono::milliseconds(
+      PROTOBUF_GET_MS_OR_DEFAULT(http2_options, graceful_goaway_timeout, 1000));
+  if (graceful_goaway_timeout_.count() > 0) {
+    graceful_goaway_timer_ =
+        connection.dispatcher().createTimer([this]() { onGracefulGoAwayTimeout(); });
+  }
 }
 
 ConnectionImpl::~ConnectionImpl() {
@@ -943,6 +952,18 @@ void ConnectionImpl::onKeepaliveResponseTimeout() {
                     StreamInfo::LocalCloseReasons::get().Http2PingTimeout);
 }
 
+void ConnectionImpl::onGracefulGoAwayTimeout() {
+  // Send final GOAWAY with actual highest received stream ID
+  if (graceful_goaway_in_progress_) {
+    ENVOY_CONN_LOG(debug, "Graceful GOAWAY timeout reached, sending final GOAWAY", connection_);
+    graceful_goaway_in_progress_ = false;
+    adapter_->SubmitGoAway(adapter_->GetHighestReceivedStreamId(),
+                           http2::adapter::Http2ErrorCode::HTTP2_NO_ERROR, "");
+    stats_.goaway_sent_.inc();
+    sendPendingFramesAndHandleError();
+  }
+}
+
 bool ConnectionImpl::slowContainsStreamId(int32_t stream_id) const {
   for (const auto& stream : active_streams_) {
     if (stream->stream_id_ == stream_id) {
@@ -1046,6 +1067,35 @@ void ConnectionImpl::goAway() {
   }
 }
 
+void ConnectionImpl::goAwayGraceful() {
+  // If graceful GOAWAY is disabled by runtime guard, fallback to immediate GOAWAY
+  if (!Runtime::runtimeFeatureEnabled("envoy.reloadable_features.http2_graceful_goaway")) {
+    goAway();
+    return;
+  }
+
+  // If graceful GOAWAY is not configured or already in progress, fallback to immediate GOAWAY
+  if (graceful_goaway_timeout_.count() == 0 || graceful_goaway_in_progress_) {
+    goAway();
+    return;
+  }
+
+  // Send initial GOAWAY with max stream ID (2^31-1) to signal graceful shutdown
+  graceful_goaway_in_progress_ = true;
+  adapter_->SubmitGoAway(0x7FFFFFFF, http2::adapter::Http2ErrorCode::HTTP2_NO_ERROR, "");
+  stats_.goaway_graceful_sent_.inc();
+
+  if (sendPendingFramesAndHandleError()) {
+    // Intended to check through coverage that this error case is tested
+    return;
+  }
+
+  // Start timer for final GOAWAY
+  if (graceful_goaway_timer_) {
+    graceful_goaway_timer_->enableTimer(graceful_goaway_timeout_);
+  }
+}
+
 void ConnectionImpl::shutdownNotice() {
   adapter_->SubmitShutdownNotice();
 
@@ -2030,7 +2080,8 @@ void ConnectionImpl::dumpState(std::ostream& os, int indent_level) const {
      << DUMP_MEMBER(max_headers_count_) << DUMP_MEMBER(per_stream_buffer_limit_)
      << DUMP_MEMBER(allow_metadata_) << DUMP_MEMBER(stream_error_on_invalid_http_messaging_)
      << DUMP_MEMBER(is_outbound_flood_monitored_control_frame_) << DUMP_MEMBER(dispatching_)
-     << DUMP_MEMBER(raised_goaway_) << DUMP_MEMBER(pending_deferred_reset_streams_.size()) << '\n';
+     << DUMP_MEMBER(raised_goaway_) << DUMP_MEMBER(graceful_goaway_in_progress_)
+     << DUMP_MEMBER(pending_deferred_reset_streams_.size()) << '\n';
 
   // Dump the protocol constraints
   DUMP_DETAILS(&protocol_constraints_);
@@ -2295,14 +2346,14 @@ Http::Status ServerConnectionImpl::dispatch(Buffer::Instance& data) {
   RETURN_IF_ERROR(protocol_constraints_.checkOutboundFrameLimits());
   if (should_send_go_away_and_close_on_dispatch_ != nullptr &&
       should_send_go_away_and_close_on_dispatch_->shouldShedLoad()) {
-    ConnectionImpl::goAway();
+    ConnectionImpl::goAwayGraceful();
     sent_go_away_on_dispatch_ = true;
     return envoyOverloadError(
         "Load shed point http2_server_go_away_and_close_on_dispatch triggered");
   }
   if (should_send_go_away_on_dispatch_ != nullptr && !sent_go_away_on_dispatch_ &&
       should_send_go_away_on_dispatch_->shouldShedLoad()) {
-    ConnectionImpl::goAway();
+    ConnectionImpl::goAwayGraceful();
     sent_go_away_on_dispatch_ = true;
   }
   return ConnectionImpl::dispatch(data);

source/common/http/http2/codec_impl.h

@@ -159,6 +159,7 @@ class ConnectionImpl : public virtual Connection,
   // NOTE: the `dispatch` method is also overridden in the ServerConnectionImpl class
   Http::Status dispatch(Buffer::Instance& data) override;
   void goAway() override;
+  void goAwayGraceful() override;
   Protocol protocol() override { return Protocol::Http2; }
   void shutdownNotice() override;
   Status protocolErrorForTest(); // Used in tests to simulate errors.
@@ -786,6 +787,7 @@ class ConnectionImpl : public virtual Connection,
                             uint32_t padding_length);
   void onKeepaliveResponse();
   void onKeepaliveResponseTimeout();
+  void onGracefulGoAwayTimeout();
   bool slowContainsStreamId(int32_t stream_id) const;
   virtual StreamResetReason getMessagingErrorResetReason() const PURE;
 
@@ -798,13 +800,16 @@ class ConnectionImpl : public virtual Connection,
   std::map<int32_t, StreamImpl*> pending_deferred_reset_streams_;
   bool dispatching_ : 1;
   bool raised_goaway_ : 1;
+  bool graceful_goaway_in_progress_ : 1;
   Event::SchedulableCallbackPtr protocol_constraint_violation_callback_;
   Random::RandomGenerator& random_;
   MonotonicTime last_received_data_time_{};
   Event::TimerPtr keepalive_send_timer_;
   Event::TimerPtr keepalive_timeout_timer_;
+  Event::TimerPtr graceful_goaway_timer_;
   std::chrono::milliseconds keepalive_interval_;
   std::chrono::milliseconds keepalive_timeout_;
+  std::chrono::milliseconds graceful_goaway_timeout_;
   uint32_t keepalive_interval_jitter_percent_;
 };
 

source/common/http/http2/codec_stats.h

@@ -16,6 +16,7 @@ namespace Http2 {
 #define ALL_HTTP2_CODEC_STATS(COUNTER, GAUGE)                                                      \
   COUNTER(dropped_headers_with_underscores)                                                        \
   COUNTER(goaway_sent)                                                                             \
+  COUNTER(goaway_graceful_sent)                                                                    \
   COUNTER(header_overflow)                                                                         \
   COUNTER(headers_cb_no_stream)                                                                    \
   COUNTER(inbound_empty_frames_flood)                                                              \

source/common/runtime/runtime_features.cc

@@ -48,6 +48,7 @@ RUNTIME_GUARD(envoy_reloadable_features_http1_balsa_allow_cr_or_lf_at_request_st
 RUNTIME_GUARD(envoy_reloadable_features_http1_balsa_delay_reset);
 RUNTIME_GUARD(envoy_reloadable_features_http1_balsa_disallow_lone_cr_in_chunk_extension);
 RUNTIME_GUARD(envoy_reloadable_features_http2_discard_host_header);
+RUNTIME_GUARD(envoy_reloadable_features_http2_graceful_goaway);
 RUNTIME_GUARD(envoy_reloadable_features_http2_propagate_reset_events);
 RUNTIME_GUARD(envoy_reloadable_features_http2_use_oghttp2);
 RUNTIME_GUARD(envoy_reloadable_features_http3_remove_empty_cookie);