From 3fb342ce5b6041ad7316bb906f2c67e0002b204c Mon Sep 17 00:00:00 2001 From: Bryan Call Date: Sat, 29 Oct 2022 13:33:47 +0100 Subject: [PATCH] HTTP/2 rate limiting Origin: backport, https://github.com/apache/trafficserver/pull/5822 Reviewed-by: Jean Baptiste Favre Last-Update: 2019-08-26 Fix for CVE-2019-9512, CVE-2019-9514, CVE-2019-9515, CVE-2019-10079 Last-Update: 2019-08-26 Gbp-Pq: Name 0015-8.0.4-CVE-backport.patch --- doc/admin-guide/files/records.config.en.rst | 28 +++ iocore/eventsystem/I_VIO.h | 6 + iocore/eventsystem/P_VIO.h | 14 ++ iocore/net/SSLNetVConnection.cc | 5 +- iocore/net/UnixNetVConnection.cc | 2 +- mgmt/RecordsConfig.cc | 8 + proxy/http2/HTTP2.cc | 37 ++-- proxy/http2/HTTP2.h | 7 + proxy/http2/Http2ClientSession.cc | 53 ++++- proxy/http2/Http2ClientSession.h | 6 + proxy/http2/Http2ConnectionState.cc | 204 +++++++++++++++++--- proxy/http2/Http2ConnectionState.h | 47 ++++- proxy/http2/Http2Stream.cc | 59 ++++++ proxy/http2/Http2Stream.h | 26 ++- 14 files changed, 445 insertions(+), 57 deletions(-) diff --git a/doc/admin-guide/files/records.config.en.rst b/doc/admin-guide/files/records.config.en.rst index a15345a0..de7ab732 100644 --- a/doc/admin-guide/files/records.config.en.rst +++ b/doc/admin-guide/files/records.config.en.rst @@ -3564,6 +3564,34 @@ HTTP/2 Configuration HTTP/2 connection to avoid duplicate pushes on the same connection. If the maximum number is reached, new entries are not remembered. +.. ts:cv:: CONFIG proxy.config.http2.max_settings_frames_per_minute INT 14 + :reloadable: + + Specifies how many SETTINGS frames |TS| receives for a minute at maximum. + Clients exceeded this limit will be immediately disconnected with an error + code of ENHANCE_YOUR_CALM. + +.. ts:cv:: CONFIG proxy.config.http2.max_ping_frames_per_minute INT 60 + :reloadable: + + Specifies how many number of PING frames |TS| receives for a minute at maximum. + Clients exceeded this limit will be immediately disconnected with an error + code of ENHANCE_YOUR_CALM. + +.. ts:cv:: CONFIG proxy.config.http2.max_priority_frames_per_minute INT 120 + :reloadable: + + Specifies how many number of PRIORITY frames |TS| receives for a minute at maximum. + Clients exceeded this limit will be immediately disconnected with an error + code of ENHANCE_YOUR_CALM. + +.. ts:cv:: CONFIG proxy.config.http2.min_avg_window_update FLOAT 2560.0 + :reloadable: + + Specifies the minimum average window increment |TS| allows. The average will be calculated based on the last 5 WINDOW_UPDATE frames. + Clients that send smaller window increments lower than this limit will be immediately disconnected with an error + code of ENHANCE_YOUR_CALM. + Plug-in Configuration ===================== diff --git a/iocore/eventsystem/I_VIO.h b/iocore/eventsystem/I_VIO.h index 6e7748c6..de7ab327 100644 --- a/iocore/eventsystem/I_VIO.h +++ b/iocore/eventsystem/I_VIO.h @@ -139,6 +139,9 @@ public: */ inkcoreapi void reenable_re(); + void disable(); + bool is_disabled(); + VIO(int aop); VIO(); @@ -218,6 +221,9 @@ public: */ Ptr mutex; + +private: + bool _disabled = false; }; #include "I_VConnection.h" diff --git a/iocore/eventsystem/P_VIO.h b/iocore/eventsystem/P_VIO.h index d8fcb054..e2bf54f1 100644 --- a/iocore/eventsystem/P_VIO.h +++ b/iocore/eventsystem/P_VIO.h @@ -104,6 +104,7 @@ VIO::set_continuation(Continuation *acont) TS_INLINE void VIO::reenable() { + this->_disabled = false; if (vc_server) { vc_server->reenable(this); } @@ -117,7 +118,20 @@ VIO::reenable() TS_INLINE void VIO::reenable_re() { + this->_disabled = false; if (vc_server) { vc_server->reenable_re(this); } } + +TS_INLINE void +VIO::disable() +{ + this->_disabled = true; +} + +TS_INLINE bool +VIO::is_disabled() +{ + return this->_disabled; +} diff --git a/iocore/net/SSLNetVConnection.cc b/iocore/net/SSLNetVConnection.cc index f9500c12..a03022c0 100644 --- a/iocore/net/SSLNetVConnection.cc +++ b/iocore/net/SSLNetVConnection.cc @@ -526,7 +526,7 @@ SSLNetVConnection::net_read_io(NetHandler *nh, EThread *lthread) // If it is not enabled, lower its priority. This allows // a fast connection to speed match a slower connection by // shifting down in priority even if it could read. - if (!s->enabled || s->vio.op != VIO::READ) { + if (!s->enabled || s->vio.op != VIO::READ || s->vio.is_disabled()) { read_disable(nh, this); return; } @@ -632,7 +632,7 @@ SSLNetVConnection::net_read_io(NetHandler *nh, EThread *lthread) } // If there is nothing to do or no space available, disable connection - if (ntodo <= 0 || !buf.writer()->write_avail()) { + if (ntodo <= 0 || !buf.writer()->write_avail() || s->vio.is_disabled()) { read_disable(nh, this); return; } @@ -1577,6 +1577,7 @@ SSLNetVConnection::reenable(NetHandler *nh) } Debug("ssl", "iterate from reenable curHook=%p %d", curHook, sslHandshakeHookState); } + this->readReschedule(nh); } diff --git a/iocore/net/UnixNetVConnection.cc b/iocore/net/UnixNetVConnection.cc index 2fd30a76..db0cc15f 100644 --- a/iocore/net/UnixNetVConnection.cc +++ b/iocore/net/UnixNetVConnection.cc @@ -201,7 +201,7 @@ read_from_net(NetHandler *nh, UnixNetVConnection *vc, EThread *thread) return; } // if it is not enabled. - if (!s->enabled || s->vio.op != VIO::READ) { + if (!s->enabled || s->vio.op != VIO::READ || s->vio.is_disabled()) { read_disable(nh, vc); return; } diff --git a/mgmt/RecordsConfig.cc b/mgmt/RecordsConfig.cc index ba886890..a8324d29 100644 --- a/mgmt/RecordsConfig.cc +++ b/mgmt/RecordsConfig.cc @@ -1330,6 +1330,14 @@ static const RecordElement RecordsConfig[] = , {RECT_LOCAL, "proxy.local.log.collation_mode", RECD_INT, "0", RECU_DYNAMIC, RR_NULL, RECC_INT, "[0-4]", RECA_NULL} , + {RECT_CONFIG, "proxy.config.http2.max_settings_frames_per_minute", RECD_INT, "14", RECU_DYNAMIC, RR_NULL, RECC_STR, "^[0-9]+$", RECA_NULL} + , + {RECT_CONFIG, "proxy.config.http2.max_ping_frames_per_minute", RECD_INT, "60", RECU_DYNAMIC, RR_NULL, RECC_STR, "^[0-9]+$", RECA_NULL} + , + {RECT_CONFIG, "proxy.config.http2.max_priority_frames_per_minute", RECD_INT, "120", RECU_DYNAMIC, RR_NULL, RECC_STR, "^[0-9]+$", RECA_NULL} + , + {RECT_CONFIG, "proxy.config.http2.min_avg_window_update", RECD_FLOAT, "2560.0", RECU_DYNAMIC, RR_NULL, RECC_NULL, nullptr, RECA_NULL} + , //# Librecords based stats system (new as of v2.1.3) {RECT_CONFIG, "proxy.config.stat_api.max_stats_allowed", RECD_INT, "256", RECU_RESTART_TS, RR_NULL, RECC_INT, "[256-1000]", RECA_NULL} diff --git a/proxy/http2/HTTP2.cc b/proxy/http2/HTTP2.cc index acfb3728..cdd4fa13 100644 --- a/proxy/http2/HTTP2.cc +++ b/proxy/http2/HTTP2.cc @@ -717,21 +717,28 @@ http2_decode_header_blocks(HTTPHdr *hdr, const uint8_t *buf_start, const uint32_ } // Initialize this subsystem with librecords configs (for now) -uint32_t Http2::max_concurrent_streams_in = 100; -uint32_t Http2::min_concurrent_streams_in = 10; -uint32_t Http2::max_active_streams_in = 0; -bool Http2::throttling = false; -uint32_t Http2::stream_priority_enabled = 0; -uint32_t Http2::initial_window_size = 1048576; -uint32_t Http2::max_frame_size = 16384; -uint32_t Http2::header_table_size = 4096; -uint32_t Http2::max_header_list_size = 4294967295; -uint32_t Http2::max_request_header_size = 131072; -uint32_t Http2::accept_no_activity_timeout = 120; -uint32_t Http2::no_activity_timeout_in = 120; -uint32_t Http2::active_timeout_in = 0; -uint32_t Http2::push_diary_size = 256; -uint32_t Http2::zombie_timeout_in = 0; +uint32_t Http2::max_concurrent_streams_in = 100; +uint32_t Http2::min_concurrent_streams_in = 10; +uint32_t Http2::max_active_streams_in = 0; +bool Http2::throttling = false; +uint32_t Http2::stream_priority_enabled = 0; +uint32_t Http2::initial_window_size = 1048576; +uint32_t Http2::max_frame_size = 16384; +uint32_t Http2::header_table_size = 4096; +uint32_t Http2::max_header_list_size = 4294967295; +uint32_t Http2::max_request_header_size = 131072; +uint32_t Http2::accept_no_activity_timeout = 120; +uint32_t Http2::no_activity_timeout_in = 120; +uint32_t Http2::active_timeout_in = 0; +uint32_t Http2::push_diary_size = 256; +uint32_t Http2::zombie_timeout_in = 0; +float Http2::stream_error_rate_threshold = 0.1; +uint32_t Http2::max_settings_per_frame = 7; +uint32_t Http2::max_settings_per_minute = 14; +uint32_t Http2::max_settings_frames_per_minute = 14; +uint32_t Http2::max_ping_frames_per_minute = 60; +uint32_t Http2::max_priority_frames_per_minute = 120; +float Http2::min_avg_window_update = 2560.0; void Http2::init() diff --git a/proxy/http2/HTTP2.h b/proxy/http2/HTTP2.h index 7b26d148..b7f822c5 100644 --- a/proxy/http2/HTTP2.h +++ b/proxy/http2/HTTP2.h @@ -378,6 +378,13 @@ public: static uint32_t active_timeout_in; static uint32_t push_diary_size; static uint32_t zombie_timeout_in; + static float stream_error_rate_threshold; + static uint32_t max_settings_per_frame; + static uint32_t max_settings_per_minute; + static uint32_t max_settings_frames_per_minute; + static uint32_t max_ping_frames_per_minute; + static uint32_t max_priority_frames_per_minute; + static float min_avg_window_update; static void init(); }; diff --git a/proxy/http2/Http2ClientSession.cc b/proxy/http2/Http2ClientSession.cc index 18b671e2..2dd46781 100644 --- a/proxy/http2/Http2ClientSession.cc +++ b/proxy/http2/Http2ClientSession.cc @@ -74,6 +74,11 @@ Http2ClientSession::destroy() void Http2ClientSession::free() { + if (this->_reenable_event) { + this->_reenable_event->cancel(); + this->_reenable_event = nullptr; + } + if (h2_pushed_urls) { this->h2_pushed_urls = ink_hash_table_destroy(this->h2_pushed_urls); } @@ -326,6 +331,13 @@ Http2ClientSession::main_event_handler(int event, void *edata) break; } + case HTTP2_SESSION_EVENT_REENABLE: + // VIO will be reenableed in this handler + retval = (this->*session_handler)(VC_EVENT_READ_READY, static_cast(e->cookie)); + // Clear the event after calling session_handler to not reschedule REENABLE in it + this->_reenable_event = nullptr; + break; + case VC_EVENT_ACTIVE_TIMEOUT: case VC_EVENT_INACTIVITY_TIMEOUT: case VC_EVENT_ERROR: @@ -478,7 +490,16 @@ Http2ClientSession::state_complete_frame_read(int event, void *edata) STATE_ENTER(&Http2ClientSession::state_complete_frame_read, event); ink_assert(event == VC_EVENT_READ_COMPLETE || event == VC_EVENT_READ_READY); if (this->sm_reader->read_avail() < this->current_hdr.length) { - vio->reenable(); + if (this->_should_do_something_else()) { + if (this->_reenable_event == nullptr) { + vio->disable(); + this->_reenable_event = mutex->thread_holding->schedule_in(this, HRTIME_MSECONDS(1), HTTP2_SESSION_EVENT_REENABLE, vio); + } else { + vio->reenable(); + } + } else { + vio->reenable(); + } return 0; } Http2SsnDebug("completed frame read, %" PRId64 " bytes available", this->sm_reader->read_avail()); @@ -495,6 +516,7 @@ Http2ClientSession::do_complete_frame_read() Http2Frame frame(this->current_hdr, this->sm_reader); send_connection_event(&this->connection_state, HTTP2_SESSION_EVENT_RECV, &frame); this->sm_reader->consume(this->current_hdr.length); + ++(this->_n_frame_read); // Set the event handler if there is no more data to process a new frame HTTP2_SET_SESSION_HANDLER(&Http2ClientSession::state_start_frame_read); @@ -510,9 +532,19 @@ Http2ClientSession::state_process_frame_read(int event, VIO *vio, bool inside_fr } while (this->sm_reader->read_avail() >= (int64_t)HTTP2_FRAME_HEADER_LEN) { + + Http2ErrorCode err = Http2ErrorCode::HTTP2_ERROR_NO_ERROR; + if (this->connection_state.get_stream_error_rate() > std::min(1.0, Http2::stream_error_rate_threshold * 2.0)) { + ip_port_text_buffer ipb; + const char *client_ip = ats_ip_ntop(get_client_addr(), ipb, sizeof(ipb)); + Error("HTTP/2 session error client_ip=%s session_id=%" PRId64 + " closing a connection, because its stream error rate (%f) is too high", + client_ip, connection_id(), this->connection_state.get_stream_error_rate()); + err = Http2ErrorCode::HTTP2_ERROR_ENHANCE_YOUR_CALM; + } + // Return if there was an error - Http2ErrorCode err; - if (do_start_frame_read(err) < 0) { + if (err > Http2ErrorCode::HTTP2_ERROR_NO_ERROR || do_start_frame_read(err) < 0) { // send an error if specified. Otherwise, just go away if (err > Http2ErrorCode::HTTP2_ERROR_NO_ERROR) { SCOPED_MUTEX_LOCK(lock, this->connection_state.mutex, this_ethread()); @@ -531,6 +563,14 @@ Http2ClientSession::state_process_frame_read(int event, VIO *vio, bool inside_fr break; } do_complete_frame_read(); + + if (this->_should_do_something_else()) { + if (this->_reenable_event == nullptr) { + vio->disable(); + this->_reenable_event = mutex->thread_holding->schedule_in(this, HRTIME_MSECONDS(1), HTTP2_SESSION_EVENT_REENABLE, vio); + return 0; + } + } } // If the client hasn't shut us down, reenable @@ -539,3 +579,10 @@ Http2ClientSession::state_process_frame_read(int event, VIO *vio, bool inside_fr } return 0; } + +bool +Http2ClientSession::_should_do_something_else() +{ + // Do something else every 128 incoming frames + return (this->_n_frame_read & 0x7F) == 0; +} diff --git a/proxy/http2/Http2ClientSession.h b/proxy/http2/Http2ClientSession.h index 998fccbd..74029417 100644 --- a/proxy/http2/Http2ClientSession.h +++ b/proxy/http2/Http2ClientSession.h @@ -42,6 +42,7 @@ #define HTTP2_SESSION_EVENT_XMIT (HTTP2_SESSION_EVENTS_START + 4) #define HTTP2_SESSION_EVENT_SHUTDOWN_INIT (HTTP2_SESSION_EVENTS_START + 5) #define HTTP2_SESSION_EVENT_SHUTDOWN_CONT (HTTP2_SESSION_EVENTS_START + 6) +#define HTTP2_SESSION_EVENT_REENABLE (HTTP2_SESSION_EVENTS_START + 7) size_t const HTTP2_HEADER_BUFFER_SIZE_INDEX = CLIENT_CONNECTION_FIRST_READ_BUFFER_SIZE_INDEX; @@ -335,6 +336,8 @@ private: // if there are multiple frames ready on the wire int state_process_frame_read(int event, VIO *vio, bool inside_frame); + bool _should_do_something_else(); + int64_t total_write_len = 0; SessionHandler session_handler = nullptr; NetVConnection *client_vc = nullptr; @@ -358,6 +361,9 @@ private: InkHashTable *h2_pushed_urls = nullptr; uint32_t h2_pushed_urls_size = 0; + + Event *_reenable_event = nullptr; + int _n_frame_read = 0; }; extern ClassAllocator http2ClientSessionAllocator; diff --git a/proxy/http2/Http2ConnectionState.cc b/proxy/http2/Http2ConnectionState.cc index a7cf6e73..af61dd85 100644 --- a/proxy/http2/Http2ConnectionState.cc +++ b/proxy/http2/Http2ConnectionState.cc @@ -27,6 +27,7 @@ #include "Http2Stream.h" #include "Http2DebugNames.h" #include +#include #define Http2ConDebug(ua_session, fmt, ...) \ SsnDebug(ua_session, "http2_con", "[%" PRId64 "] " fmt, ua_session->connection_id(), ##__VA_ARGS__); @@ -137,18 +138,18 @@ rcv_data_frame(Http2ConnectionState &cstate, const Http2Frame &frame) } // Check whether Window Size is acceptable - if (cstate.server_rwnd < payload_length) { + if (cstate.server_rwnd() < payload_length) { return Http2Error(Http2ErrorClass::HTTP2_ERROR_CLASS_CONNECTION, Http2ErrorCode::HTTP2_ERROR_FLOW_CONTROL_ERROR, "recv data cstate.server_rwnd < payload_length"); } - if (stream->server_rwnd < payload_length) { + if (stream->server_rwnd() < payload_length) { return Http2Error(Http2ErrorClass::HTTP2_ERROR_CLASS_STREAM, Http2ErrorCode::HTTP2_ERROR_FLOW_CONTROL_ERROR, "recv data stream->server_rwnd < payload_length"); } // Update Window size - cstate.server_rwnd -= payload_length; - stream->server_rwnd -= payload_length; + cstate.decrement_server_rwnd(payload_length); + stream->decrement_server_rwnd(payload_length); const uint32_t unpadded_length = payload_length - pad_length; // If we call write() multiple times, we must keep the same reader, so we can @@ -174,15 +175,15 @@ rcv_data_frame(Http2ConnectionState &cstate, const Http2Frame &frame) uint32_t initial_rwnd = cstate.server_settings.get(HTTP2_SETTINGS_INITIAL_WINDOW_SIZE); uint32_t min_rwnd = std::min(initial_rwnd, cstate.server_settings.get(HTTP2_SETTINGS_MAX_FRAME_SIZE)); // Connection level WINDOW UPDATE - if (cstate.server_rwnd <= min_rwnd) { - Http2WindowSize diff_size = initial_rwnd - cstate.server_rwnd; - cstate.server_rwnd += diff_size; + if (cstate.server_rwnd() <= min_rwnd) { + Http2WindowSize diff_size = initial_rwnd - cstate.server_rwnd(); + cstate.increment_server_rwnd(diff_size); cstate.send_window_update_frame(0, diff_size); } // Stream level WINDOW UPDATE - if (stream->server_rwnd <= min_rwnd) { - Http2WindowSize diff_size = initial_rwnd - stream->server_rwnd; - stream->server_rwnd += diff_size; + if (stream->server_rwnd() <= min_rwnd) { + Http2WindowSize diff_size = initial_rwnd - stream->server_rwnd(); + stream->increment_server_rwnd(diff_size); cstate.send_window_update_frame(stream->get_id(), diff_size); } @@ -406,6 +407,17 @@ rcv_priority_frame(Http2ConnectionState &cstate, const Http2Frame &frame) "PRIORITY frame depends on itself"); } + // Update PRIORITY frame count per minute + cstate.increment_received_priority_frame_count(); + // Close this conection if its priority frame count received exceeds a limit + if (cstate.get_received_priority_frame_count() > Http2::max_priority_frames_per_minute) { + Http2StreamDebug(cstate.ua_session, stream_id, + "Observed too frequent priority changes: %u priority changes within a last minute", + cstate.get_received_priority_frame_count()); + return Http2Error(Http2ErrorClass::HTTP2_ERROR_CLASS_CONNECTION, Http2ErrorCode::HTTP2_ERROR_ENHANCE_YOUR_CALM, + "recv priority too frequent priority changes"); + } + if (!Http2::stream_priority_enabled) { return Http2Error(Http2ErrorClass::HTTP2_ERROR_CLASS_NONE); } @@ -511,6 +523,16 @@ rcv_settings_frame(Http2ConnectionState &cstate, const Http2Frame &frame) Warning("Setting frame for zombied sessoin %" PRId64, cstate.ua_session->connection_id()); } + // Update SETTIGNS frame count per minute + cstate.increment_received_settings_frame_count(); + // Close this conection if its SETTINGS frame count exceeds a limit + if (cstate.get_received_settings_frame_count() > Http2::max_settings_frames_per_minute) { + Http2StreamDebug(cstate.ua_session, stream_id, "Observed too frequent SETTINGS frames: %u frames within a last minute", + cstate.get_received_settings_frame_count()); + return Http2Error(Http2ErrorClass::HTTP2_ERROR_CLASS_CONNECTION, Http2ErrorCode::HTTP2_ERROR_ENHANCE_YOUR_CALM, + "recv settings too frequent SETTINGS frames"); + } + // [RFC 7540] 6.5. The stream identifier for a SETTINGS frame MUST be zero. // If an endpoint receives a SETTINGS frame whose stream identifier field is // anything other than 0x0, the endpoint MUST respond with a connection @@ -614,6 +636,16 @@ rcv_ping_frame(Http2ConnectionState &cstate, const Http2Frame &frame) "ping bad length"); } + // Update PING frame count per minute + cstate.increment_received_ping_frame_count(); + // Close this conection if its ping count received exceeds a limit + if (cstate.get_received_ping_frame_count() > Http2::max_ping_frames_per_minute) { + Http2StreamDebug(cstate.ua_session, stream_id, "Observed too frequent PING frames: %u PING frames within a last minute", + cstate.get_received_ping_frame_count()); + return Http2Error(Http2ErrorClass::HTTP2_ERROR_CLASS_CONNECTION, Http2ErrorCode::HTTP2_ERROR_ENHANCE_YOUR_CALM, + "recv ping too frequent PING frame"); + } + // An endpoint MUST NOT respond to PING frames containing this flag. if (frame.header().flags & HTTP2_FLAGS_PING_ACK) { return Http2Error(Http2ErrorClass::HTTP2_ERROR_CLASS_NONE); @@ -695,7 +727,7 @@ rcv_window_update_frame(Http2ConnectionState &cstate, const Http2Frame &frame) if (stream_id == 0) { // Connection level window update Http2StreamDebug(cstate.ua_session, stream_id, "Received WINDOW_UPDATE frame - updated to: %zd delta: %u", - (cstate.client_rwnd + size), size); + (cstate.client_rwnd() + size), size); // A sender MUST NOT allow a flow-control window to exceed 2^31-1 // octets. If a sender receives a WINDOW_UPDATE that causes a flow- @@ -704,12 +736,16 @@ rcv_window_update_frame(Http2ConnectionState &cstate, const Http2Frame &frame) // sends a RST_STREAM with an error code of FLOW_CONTROL_ERROR; for the // connection, a GOAWAY frame with an error code of FLOW_CONTROL_ERROR // is sent. - if (size > HTTP2_MAX_WINDOW_SIZE - cstate.client_rwnd) { + if (size > HTTP2_MAX_WINDOW_SIZE - cstate.client_rwnd()) { return Http2Error(Http2ErrorClass::HTTP2_ERROR_CLASS_CONNECTION, Http2ErrorCode::HTTP2_ERROR_FLOW_CONTROL_ERROR, "window update too big"); } - cstate.client_rwnd += size; + auto error = cstate.increment_client_rwnd(size); + if (error != Http2ErrorCode::HTTP2_ERROR_NO_ERROR) { + return Http2Error(Http2ErrorClass::HTTP2_ERROR_CLASS_CONNECTION, error); + } + cstate.restart_streams(); } else { // Stream level window update @@ -725,7 +761,7 @@ rcv_window_update_frame(Http2ConnectionState &cstate, const Http2Frame &frame) } Http2StreamDebug(cstate.ua_session, stream_id, "Received WINDOW_UPDATE frame - updated to: %zd delta: %u", - (stream->client_rwnd + size), size); + (stream->client_rwnd() + size), size); // A sender MUST NOT allow a flow-control window to exceed 2^31-1 // octets. If a sender receives a WINDOW_UPDATE that causes a flow- @@ -734,14 +770,17 @@ rcv_window_update_frame(Http2ConnectionState &cstate, const Http2Frame &frame) // sends a RST_STREAM with an error code of FLOW_CONTROL_ERROR; for the // connection, a GOAWAY frame with an error code of FLOW_CONTROL_ERROR // is sent. - if (size > HTTP2_MAX_WINDOW_SIZE - stream->client_rwnd) { + if (size > HTTP2_MAX_WINDOW_SIZE - stream->client_rwnd()) { return Http2Error(Http2ErrorClass::HTTP2_ERROR_CLASS_STREAM, Http2ErrorCode::HTTP2_ERROR_FLOW_CONTROL_ERROR, "window update too big 2"); } - stream->client_rwnd += size; - ssize_t wnd = std::min(cstate.client_rwnd, stream->client_rwnd); + auto error = stream->increment_client_rwnd(size); + if (error != Http2ErrorCode::HTTP2_ERROR_NO_ERROR) { + return Http2Error(Http2ErrorClass::HTTP2_ERROR_CLASS_STREAM, error); + } + ssize_t wnd = std::min(cstate.client_rwnd(), stream->client_rwnd()); if (!stream->is_closed() && stream->get_state() == Http2StreamState::HTTP2_STREAM_STATE_HALF_CLOSED_REMOTE && wnd > 0) { SCOPED_MUTEX_LOCK(lock, stream->mutex, this_ethread()); stream->restart_sending(); @@ -1137,7 +1176,7 @@ Http2ConnectionState::restart_streams() while (s != end) { Http2Stream *next = static_cast(s->link.next ? s->link.next : stream_list.head); if (!s->is_closed() && s->get_state() == Http2StreamState::HTTP2_STREAM_STATE_HALF_CLOSED_REMOTE && - std::min(this->client_rwnd, s->client_rwnd) > 0) { + std::min(this->client_rwnd(), s->client_rwnd()) > 0) { SCOPED_MUTEX_LOCK(lock, s->mutex, this_ethread()); s->restart_sending(); } @@ -1145,7 +1184,7 @@ Http2ConnectionState::restart_streams() s = next; } if (!s->is_closed() && s->get_state() == Http2StreamState::HTTP2_STREAM_STATE_HALF_CLOSED_REMOTE && - std::min(this->client_rwnd, s->client_rwnd) > 0) { + std::min(this->client_rwnd(), s->client_rwnd()) > 0) { SCOPED_MUTEX_LOCK(lock, s->mutex, this_ethread()); s->restart_sending(); } @@ -1284,7 +1323,7 @@ Http2ConnectionState::update_initial_rwnd(Http2WindowSize new_size) // Update stream level window sizes for (Http2Stream *s = stream_list.head; s; s = static_cast(s->link.next)) { SCOPED_MUTEX_LOCK(lock, s->mutex, this_ethread()); - s->client_rwnd = new_size - (client_settings.get(HTTP2_SETTINGS_INITIAL_WINDOW_SIZE) - s->client_rwnd); + s->update_initial_rwnd(new_size - (client_settings.get(HTTP2_SETTINGS_INITIAL_WINDOW_SIZE) - s->client_rwnd())); } } @@ -1313,7 +1352,7 @@ Http2ConnectionState::send_data_frames_depends_on_priority() Http2DependencyTree::Node *node = dependency_tree->top(); // No node to send or no connection level window left - if (node == nullptr || client_rwnd <= 0) { + if (node == nullptr || _client_rwnd <= 0) { return; } @@ -1355,7 +1394,7 @@ Http2ConnectionState::send_data_frames_depends_on_priority() Http2SendDataFrameResult Http2ConnectionState::send_a_data_frame(Http2Stream *stream, size_t &payload_length) { - const ssize_t window_size = std::min(this->client_rwnd, stream->client_rwnd); + const ssize_t window_size = std::min(this->client_rwnd(), stream->client_rwnd()); const size_t buf_len = BUFFER_SIZE_FOR_INDEX(buffer_size_index[HTTP2_FRAME_TYPE_DATA]); const size_t write_available_size = std::min(buf_len, static_cast(window_size)); size_t read_available_size = 0; @@ -1401,12 +1440,12 @@ Http2ConnectionState::send_a_data_frame(Http2Stream *stream, size_t &payload_len } // Update window size - this->client_rwnd -= payload_length; - stream->client_rwnd -= payload_length; + this->decrement_client_rwnd(payload_length); + stream->decrement_client_rwnd(payload_length); // Create frame Http2StreamDebug(ua_session, stream->get_id(), "Send a DATA frame - client window con: %5zd stream: %5zd payload: %5zd", - client_rwnd, stream->client_rwnd, payload_length); + _client_rwnd, stream->client_rwnd(), payload_length); Http2Frame data(HTTP2_FRAME_TYPE_DATA, stream->get_id(), flags); data.alloc(buffer_size_index[HTTP2_FRAME_TYPE_DATA]); @@ -1817,6 +1856,72 @@ Http2ConnectionState::send_window_update_frame(Http2StreamId id, uint32_t size) this->ua_session->handleEvent(HTTP2_SESSION_EVENT_XMIT, &window_update); } +void +Http2ConnectionState::increment_received_settings_frame_count() +{ + ink_hrtime hrtime_sec = Thread::get_hrtime() / HRTIME_SECOND; + uint8_t counter_index = ((hrtime_sec % 60) >= 30); + + if ((hrtime_sec - 60) > this->settings_frame_count_last_update) { + this->settings_frame_count[0] = 0; + this->settings_frame_count[1] = 0; + } else if (counter_index != ((this->settings_frame_count_last_update % 60) >= 30)) { + this->settings_frame_count[counter_index] = 0; + } + ++this->settings_frame_count[counter_index]; + this->settings_frame_count_last_update = hrtime_sec; +} + +uint32_t +Http2ConnectionState::get_received_settings_frame_count() +{ + return this->settings_frame_count[0] + this->settings_frame_count[1]; +} + +void +Http2ConnectionState::increment_received_ping_frame_count() +{ + ink_hrtime hrtime_sec = Thread::get_hrtime() / HRTIME_SECOND; + uint8_t counter_index = ((hrtime_sec % 60) >= 30); + + if ((hrtime_sec - 60) > this->ping_frame_count_last_update) { + this->ping_frame_count[0] = 0; + this->ping_frame_count[1] = 0; + } else if (counter_index != ((this->ping_frame_count_last_update % 60) >= 30)) { + this->ping_frame_count[counter_index] = 0; + } + ++this->ping_frame_count[counter_index]; + this->ping_frame_count_last_update = hrtime_sec; +} + +uint32_t +Http2ConnectionState::get_received_ping_frame_count() +{ + return this->ping_frame_count[0] + this->ping_frame_count[1]; +} + +void +Http2ConnectionState::increment_received_priority_frame_count() +{ + ink_hrtime hrtime_sec = Thread::get_hrtime() / HRTIME_SECOND; + uint8_t counter_index = ((hrtime_sec % 60) >= 30); + + if ((hrtime_sec - 60) > this->priority_frame_count_last_update) { + this->priority_frame_count[0] = 0; + this->priority_frame_count[1] = 0; + } else if (counter_index != ((this->priority_frame_count_last_update % 60) >= 30)) { + this->priority_frame_count[counter_index] = 0; + } + ++this->priority_frame_count[counter_index]; + this->priority_frame_count_last_update = hrtime_sec; +} + +uint32_t +Http2ConnectionState::get_received_priority_frame_count() +{ + return this->priority_frame_count[0] + this->priority_frame_count[1]; +} + // Return min_concurrent_streams_in when current client streams number is larger than max_active_streams_in. // Main purpose of this is preventing DDoS Attacks. unsigned @@ -1849,3 +1954,52 @@ Http2ConnectionState::_adjust_concurrent_stream() return Http2::max_concurrent_streams_in; } + +ssize_t +Http2ConnectionState::client_rwnd() const +{ + return this->_client_rwnd; +} + +Http2ErrorCode +Http2ConnectionState::increment_client_rwnd(size_t amount) +{ + this->_client_rwnd += amount; + + this->_recent_rwnd_increment[this->_recent_rwnd_increment_index] = amount; + ++this->_recent_rwnd_increment_index; + this->_recent_rwnd_increment_index %= this->_recent_rwnd_increment.size(); + double sum = std::accumulate(this->_recent_rwnd_increment.begin(), this->_recent_rwnd_increment.end(), 0.0); + double avg = sum / this->_recent_rwnd_increment.size(); + if (avg < Http2::min_avg_window_update) { + return Http2ErrorCode::HTTP2_ERROR_ENHANCE_YOUR_CALM; + } + return Http2ErrorCode::HTTP2_ERROR_NO_ERROR; +} + +Http2ErrorCode +Http2ConnectionState::decrement_client_rwnd(size_t amount) +{ + this->_client_rwnd -= amount; + return Http2ErrorCode::HTTP2_ERROR_NO_ERROR; +} + +ssize_t +Http2ConnectionState::server_rwnd() const +{ + return this->_server_rwnd; +} + +Http2ErrorCode +Http2ConnectionState::increment_server_rwnd(size_t amount) +{ + this->_server_rwnd += amount; + return Http2ErrorCode::HTTP2_ERROR_NO_ERROR; +} + +Http2ErrorCode +Http2ConnectionState::decrement_server_rwnd(size_t amount) +{ + this->_server_rwnd -= amount; + return Http2ErrorCode::HTTP2_ERROR_NO_ERROR; +} diff --git a/proxy/http2/Http2ConnectionState.h b/proxy/http2/Http2ConnectionState.h index ef94a933..eabe5566 100644 --- a/proxy/http2/Http2ConnectionState.h +++ b/proxy/http2/Http2ConnectionState.h @@ -220,9 +220,16 @@ public: return client_streams_in_count; } - // Connection level window size - ssize_t client_rwnd = HTTP2_INITIAL_WINDOW_SIZE; - ssize_t server_rwnd = Http2::initial_window_size; + double + get_stream_error_rate() const + { + int total = get_stream_requests(); + if (total > 0) { + return (double)stream_error_count / (double)total; + } else { + return 0; + } + } // HTTP/2 frame sender void schedule_stream(Http2Stream *stream); @@ -292,6 +299,20 @@ public: } } + void increment_received_settings_frame_count(); + uint32_t get_received_settings_frame_count(); + void increment_received_ping_frame_count(); + uint32_t get_received_ping_frame_count(); + void increment_received_priority_frame_count(); + uint32_t get_received_priority_frame_count(); + + ssize_t client_rwnd() const; + Http2ErrorCode increment_client_rwnd(size_t amount); + Http2ErrorCode decrement_client_rwnd(size_t amount); + ssize_t server_rwnd() const; + Http2ErrorCode increment_server_rwnd(size_t amount); + Http2ErrorCode decrement_server_rwnd(size_t amount); + private: unsigned _adjust_concurrent_stream(); @@ -330,4 +351,24 @@ private: Event *shutdown_cont_event = nullptr; Event *fini_event = nullptr; Event *zombie_event = nullptr; + + // Counter for stream errors ATS sent + uint32_t stream_error_count; + + // Connection level window size + ssize_t _client_rwnd = HTTP2_INITIAL_WINDOW_SIZE; + ssize_t _server_rwnd = Http2::initial_window_size; + + std::vector _recent_rwnd_increment = {SIZE_MAX, SIZE_MAX, SIZE_MAX, SIZE_MAX, SIZE_MAX}; + int _recent_rwnd_increment_index = 0; + + // Counters for frames received within last 60 seconds + // Each item in an array holds a count for 30 seconds. + uint16_t settings_frame_count[2] = {0}; + ink_hrtime settings_frame_count_last_update = 0; + uint16_t ping_frame_count[2] = {0}; + ink_hrtime ping_frame_count_last_update = 0; + uint16_t priority_frame_count[2] = {0}; + ink_hrtime priority_frame_count_last_update = 0; + }; diff --git a/proxy/http2/Http2Stream.cc b/proxy/http2/Http2Stream.cc index 1f3cd887..2f91c0f2 100644 --- a/proxy/http2/Http2Stream.cc +++ b/proxy/http2/Http2Stream.cc @@ -26,6 +26,8 @@ #include "Http2ClientSession.h" #include "../http/HttpSM.h" +#include + #define Http2StreamDebug(fmt, ...) \ SsnDebug(parent, "http2_stream", "[%" PRId64 "] [%u] " fmt, parent->connection_id(), this->get_id(), ##__VA_ARGS__); @@ -812,6 +814,63 @@ Http2Stream::response_process_data(bool &done) } } +ssize_t +Http2Stream::client_rwnd() const +{ + return this->_client_rwnd; +} + +Http2ErrorCode +Http2Stream::increment_client_rwnd(size_t amount) +{ + this->_client_rwnd += amount; + + this->_recent_rwnd_increment[this->_recent_rwnd_increment_index] = amount; + ++this->_recent_rwnd_increment_index; + this->_recent_rwnd_increment_index %= this->_recent_rwnd_increment.size(); + double sum = std::accumulate(this->_recent_rwnd_increment.begin(), this->_recent_rwnd_increment.end(), 0.0); + double avg = sum / this->_recent_rwnd_increment.size(); + if (avg < Http2::min_avg_window_update) { + return Http2ErrorCode::HTTP2_ERROR_ENHANCE_YOUR_CALM; + } + return Http2ErrorCode::HTTP2_ERROR_NO_ERROR; +} + +Http2ErrorCode +Http2Stream::decrement_client_rwnd(size_t amount) +{ + this->_client_rwnd -= amount; + if (this->_client_rwnd < 0) { + return Http2ErrorCode::HTTP2_ERROR_PROTOCOL_ERROR; + } else { + return Http2ErrorCode::HTTP2_ERROR_NO_ERROR; + } +} + +ssize_t +Http2Stream::server_rwnd() const +{ + return this->_server_rwnd; +} + +Http2ErrorCode +Http2Stream::increment_server_rwnd(size_t amount) +{ + this->_server_rwnd += amount; + return Http2ErrorCode::HTTP2_ERROR_NO_ERROR; +} + +Http2ErrorCode +Http2Stream::decrement_server_rwnd(size_t amount) +{ + this->_server_rwnd -= amount; + if (this->_server_rwnd < 0) { + return Http2ErrorCode::HTTP2_ERROR_PROTOCOL_ERROR; + } else { + return Http2ErrorCode::HTTP2_ERROR_NO_ERROR; + } +} + bool Http2Stream::response_is_data_available() const { diff --git a/proxy/http2/Http2Stream.h b/proxy/http2/Http2Stream.h index 1fecf80d..31e08470 100644 --- a/proxy/http2/Http2Stream.h +++ b/proxy/http2/Http2Stream.h @@ -38,7 +38,7 @@ class Http2Stream : public ProxyClientTransaction { public: typedef ProxyClientTransaction super; ///< Parent type. - Http2Stream(Http2StreamId sid = 0, ssize_t initial_rwnd = Http2::initial_window_size) : client_rwnd(initial_rwnd), _id(sid) + Http2Stream(Http2StreamId sid = 0, ssize_t initial_rwnd = Http2::initial_window_size) : _id(sid), _client_rwnd(initial_rwnd) { SET_HANDLER(&Http2Stream::main_event_handler); } @@ -46,10 +46,10 @@ public: void init(Http2StreamId sid, ssize_t initial_rwnd) { - _id = sid; - _start_time = Thread::get_hrtime(); - _thread = this_ethread(); - this->client_rwnd = initial_rwnd; + _id = sid; + _start_time = Thread::get_hrtime(); + _thread = this_ethread(); + this->_client_rwnd = initial_rwnd; HTTP2_INCREMENT_THREAD_DYN_STAT(HTTP2_STAT_CURRENT_CLIENT_STREAM_COUNT, _thread); HTTP2_INCREMENT_THREAD_DYN_STAT(HTTP2_STAT_TOTAL_CLIENT_STREAM_COUNT, _thread); sm_reader = request_reader = request_buffer.alloc_reader(); @@ -117,7 +117,7 @@ public: void update_initial_rwnd(Http2WindowSize new_size) { - client_rwnd = new_size; + this->_client_rwnd = new_size; } bool @@ -171,8 +171,12 @@ public: void push_promise(URL &url, const MIMEField *accept_encoding); // Stream level window size - ssize_t client_rwnd; - ssize_t server_rwnd = Http2::initial_window_size; + ssize_t client_rwnd() const; + Http2ErrorCode increment_client_rwnd(size_t amount); + Http2ErrorCode decrement_client_rwnd(size_t amount); + ssize_t server_rwnd() const; + Http2ErrorCode increment_server_rwnd(size_t amount); + Http2ErrorCode decrement_server_rwnd(size_t amount); uint8_t *header_blocks = nullptr; uint32_t header_blocks_length = 0; // total length of header blocks (not include @@ -288,6 +292,12 @@ private: uint64_t data_length = 0; uint64_t bytes_sent = 0; + ssize_t _client_rwnd; + ssize_t _server_rwnd = Http2::initial_window_size; + + std::vector _recent_rwnd_increment = {SIZE_MAX, SIZE_MAX, SIZE_MAX, SIZE_MAX, SIZE_MAX}; + int _recent_rwnd_increment_index = 0; + ChunkedHandler chunked_handler; Event *cross_thread_event = nullptr; Event *buffer_full_write_event = nullptr; -- 2.30.2