From b96d7e7becda2d47feb6dc22f8b43ba7b987dc8a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9D=8E=E9=80=9A=E6=B4=B2?= Date: Wed, 16 Sep 2026 00:56:21 +0800 Subject: [PATCH] Networking: supports chunked response & custom port number --- CMakeLists.txt | 8 + src/common/impl/networking_common.c | 209 +++++++++++++++++++++++++++ src/common/impl/networking_linux.c | 155 ++++++++++++++++---- src/common/impl/networking_windows.c | 96 ++++++++++-- src/common/networking.h | 34 ++++- src/detection/publicip/publicip.c | 50 ++++++- src/detection/weather/weather.c | 2 +- tests/networking.c | 86 +++++++++++ 8 files changed, 600 insertions(+), 40 deletions(-) create mode 100644 tests/networking.c diff --git a/CMakeLists.txt b/CMakeLists.txt index 25e2e4dc0..6f09152a8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -2204,6 +2204,13 @@ if (BUILD_TESTS) PRIVATE libfastfetch ) + add_executable(fastfetch-test-networking + tests/networking.c + ) + target_link_libraries(fastfetch-test-networking + PRIVATE libfastfetch + ) + enable_testing() add_test(NAME test-strbuf COMMAND fastfetch-test-strbuf) add_test(NAME test-list COMMAND fastfetch-test-list) @@ -2211,6 +2218,7 @@ if (BUILD_TESTS) add_test(NAME test-color COMMAND fastfetch-test-color) add_test(NAME test-duration COMMAND fastfetch-test-duration) add_test(NAME test-strutil COMMAND fastfetch-test-strutil) + add_test(NAME test-networking COMMAND fastfetch-test-networking) endif() ################## diff --git a/src/common/impl/networking_common.c b/src/common/impl/networking_common.c index 2c436590e..cc6f5e24b 100644 --- a/src/common/impl/networking_common.c +++ b/src/common/impl/networking_common.c @@ -190,3 +190,212 @@ bool ffNetworkingDecompressGzip(FFstrbuf* buffer, char* headerEnd) { return true; } #endif // FF_HAVE_ZLIB + +const char* ffNetworkingFindHeader(const char* headers, uint32_t headerEnd, const char* name, uint32_t* valueLen) { + assert(headers != nullptr && valueLen != nullptr); + + uint32_t nameLen = (uint32_t) strlen(name); + uint32_t pos = 0; + + while (pos < headerEnd) { + uint32_t eol = pos; + while (eol < headerEnd && headers[eol] != '\n') { + ++eol; + } + uint32_t lineEnd = (eol > pos && headers[eol - 1] == '\r') ? (eol - 1) : eol; + + // Only match at the beginning of a line, so that a value can never be mistaken + // for a field name (obs-fold continuation lines included) + if (lineEnd - pos >= nameLen && strncasecmp(headers + pos, name, nameLen) == 0) { + uint32_t valueStart = pos + nameLen; + while (valueStart < lineEnd && (headers[valueStart] == ' ' || headers[valueStart] == '\t')) { + ++valueStart; + } + *valueLen = lineEnd - valueStart; + return headers + valueStart; + } + + if (eol >= headerEnd) { + break; + } + pos = eol + 1; + } + + return nullptr; +} + +FFNetworkingTransferEncoding ffNetworkingParseTransferEncoding(const char* value, uint32_t valueLen) { + assert(value != nullptr); + + // The value is a comma-separated list of transfer codings (RFC 9112 6.1) + uint32_t codingCount = 0; + const char* coding = nullptr; + uint32_t codingLen = 0; + + for (uint32_t i = 0, start = 0; i <= valueLen; ++i) { + if (i < valueLen && value[i] != ',') { + continue; + } + + // trim the optional whitespace around the coding + uint32_t from = start; + uint32_t to = i; + while (from < to && (value[from] == ' ' || value[from] == '\t')) { + ++from; + } + while (to > from && (value[to - 1] == ' ' || value[to - 1] == '\t')) { + --to; + } + + if (to > from) { + ++codingCount; + coding = value + from; + codingLen = to - from; + } + start = i + 1; + } + + if (codingCount == 0) { + return FF_NETWORKING_TE_NONE; + } + + if (codingCount == 1 && codingLen == 7 && strncasecmp(coding, "chunked", 7) == 0) { + return FF_NETWORKING_TE_CHUNKED; + } + + return FF_NETWORKING_TE_UNSUPPORTED; +} + +int ffNetworkingChunkedComplete(const char* body, uint32_t bodyLen, uint32_t* consumed) { + assert(body != nullptr && consumed != nullptr); + + uint32_t pos = 0; + + for (;;) { + // chunk-size [ chunk-ext ] CRLF + uint32_t eol = pos; + while (eol < bodyLen && body[eol] != '\n') { + ++eol; + } + if (eol >= bodyLen) { + return 0; // the chunk-size line is not complete yet + } + + if (eol == pos || !isxdigit((unsigned char) body[pos])) { + return -1; + } + + char* stop = nullptr; + unsigned long size = strtoul(body + pos, &stop, 16); + if (stop == body + pos) { + return -1; + } + pos = eol + 1; // skips the chunk extension, which ends at the CRLF + + if (size == 0) { + // last-chunk, followed by an optional trailer section and an empty line + uint32_t p = pos; + while (p < bodyLen) { + uint32_t e = p; + while (e < bodyLen && body[e] != '\n') { + ++e; + } + if (e >= bodyLen) { + return 0; + } + if (e == p || (e == p + 1 && body[p] == '\r')) { + *consumed = e + 1; + return 1; + } + p = e + 1; + } + return 0; + } + + if (size > (unsigned long) (bodyLen - pos)) { + return 0; // the chunk data is not complete yet + } + pos += (uint32_t) size; + + if (bodyLen - pos < 2) { + return 0; + } + if (body[pos] != '\r' || body[pos + 1] != '\n') { + return -1; + } + pos += 2; + } +} + +// Keeps the status line and every header except `dropHeader` and `Content-Length`, +// then emits a `Content-Length` matching the (already decoded) body. +// `body` may point into `buffer->chars`; it is copied before `buffer` is released. +static void rebuildResponse(FFstrbuf* buffer, uint32_t headerEnd, const char* body, uint32_t bodyLen, const char* dropHeader) { + FF_STRBUF_AUTO_DESTROY newBuffer = ffStrbufCreateA(headerEnd + bodyLen + 64); + + uint32_t pos = 0; + while (pos < headerEnd) { + uint32_t eol = pos; + while (eol < headerEnd && buffer->chars[eol] != '\n') { + ++eol; + } + uint32_t lineLen = (eol < headerEnd) ? (eol - pos + 1) : (headerEnd - pos); + + if (!ffStrStartsWithIgnCase(buffer->chars + pos, "Content-Length:") && + !ffStrStartsWithIgnCase(buffer->chars + pos, dropHeader)) { + ffStrbufAppendNS(&newBuffer, lineLen, buffer->chars + pos); + } + + pos += lineLen; + } + + ffStrbufAppendF(&newBuffer, "Content-Length: %u\r\n\r\n", bodyLen); + ffStrbufAppendNS(&newBuffer, bodyLen, body); + + ffStrbufDestroy(buffer); + ffStrbufInitMove(buffer, &newBuffer); +} + +bool ffNetworkingDecodeChunked(FFstrbuf* buffer, uint32_t headerEnd) { + assert(buffer->allocated > 0); + + // The header block is terminated by CR LF CR LF + if (headerEnd + 4 > buffer->length) { + return false; + } + + char* body = buffer->chars + headerEnd + 4; + uint32_t bodyLen = buffer->length - headerEnd - 4; + + uint32_t consumed = 0; + if (ffNetworkingChunkedComplete(body, bodyLen, &consumed) != 1) { + FF_DEBUG("Incomplete or malformed chunked body"); + return false; + } + + // Decoding in place is safe: the encoded form is never shorter than the payload + char* out = body; + uint32_t outLen = 0; + uint32_t pos = 0; + + while (pos < bodyLen) { + uint32_t eol = pos; + while (eol < bodyLen && body[eol] != '\n') { + ++eol; + } + + unsigned long size = strtoul(body + pos, nullptr, 16); + pos = eol + 1; + if (size == 0) { + break; // last-chunk; trailers are dropped + } + + memmove(out + outLen, body + pos, size); + outLen += (uint32_t) size; + pos += (uint32_t) size + 2; // trailing CRLF + } + + FF_DEBUG("Decoded chunked body: %u bytes encoded, %u bytes decoded", bodyLen, outLen); + rebuildResponse(buffer, headerEnd, out, outLen, "Transfer-Encoding:"); + return true; +} diff --git a/src/common/impl/networking_linux.c b/src/common/impl/networking_linux.c index 770c0ce77..c9e708e24 100644 --- a/src/common/impl/networking_linux.c +++ b/src/common/impl/networking_linux.c @@ -16,6 +16,9 @@ #include #include +// Upper bound of a single HTTP response, guarding against excessive memory allocation +#define FF_NETWORKING_MAX_RESPONSE_SIZE (1024u * 1024u) + static const char* tryNonThreadingFastPath(FFNetworkingState* state) { #if defined(TCP_FASTOPEN) || __APPLE__ @@ -187,7 +190,7 @@ exit: FF_THREAD_ENTRY_DECL_WRAPPER(connectAndSend, FFNetworkingState*); // Parallel DNS resolution and socket creation -static const char* initNetworkingState(FFNetworkingState* state, const char* host, const char* path, const char* headers) { +static const char* initNetworkingState(FFNetworkingState* state, const char* host, uint16_t port, const char* path, const char* headers) { FF_DEBUG("Initializing network connection state: host=%s, path=%s", host, path); // Initialize command and host information @@ -195,7 +198,19 @@ static const char* initNetworkingState(FFNetworkingState* state, const char* hos ffStrbufAppendS(&state->command, "GET "); ffStrbufAppendS(&state->command, path); ffStrbufAppendS(&state->command, " HTTP/1.0\r\nHost: "); - ffStrbufAppendS(&state->command, host); + if (strchr(host, ':') != nullptr) { + // An IPv6 literal has to be bracketed in the Host header (RFC 9110 7.2), while + // getaddrinfo() wants it bare + ffStrbufAppendC(&state->command, '['); + ffStrbufAppendS(&state->command, host); + ffStrbufAppendC(&state->command, ']'); + } else { + ffStrbufAppendS(&state->command, host); + } + // The Host header carries the port whenever it is not the default one (RFC 9110 7.2) + if (port != 80) { + ffStrbufAppendF(&state->command, ":%u", port); + } ffStrbufAppendS(&state->command, "\r\nConnection: close\r\n"); // Explicitly tell the server we don't need to keep the connection // If compression needs to be enabled @@ -220,10 +235,13 @@ static const char* initNetworkingState(FFNetworkingState* state, const char* hos .ai_flags = AI_NUMERICSERV }; - FF_DEBUG("Resolving address: %s (%s)", host, state->ipv6 ? "IPv6" : "IPv4"); + char portA[6]; + snprintf(portA, sizeof(portA), "%u", port); + + FF_DEBUG("Resolving address: %s:%u (%s)", host, port, state->ipv6 ? "IPv6" : "IPv4"); // Use AI_NUMERICSERV flag to indicate the service is a numeric port, reducing parsing time - int gaiRes = getaddrinfo(host, "80", &hints, &state->addr); + int gaiRes = getaddrinfo(host, portA, &hints, &state->addr); if (gaiRes != 0) { FF_DEBUG("getaddrinfo() failed: %s (res=%d)", gai_strerror(gaiRes), gaiRes); ret = "getaddrinfo() failed"; @@ -259,6 +277,15 @@ static const char* initNetworkingState(FFNetworkingState* state, const char* hos } #endif +#ifdef SO_NOSIGPIPE + // Prevent SIGPIPE when the server closes the connection during write + if (setsockopt(state->sockfd, SOL_SOCKET, SO_NOSIGPIPE, &flag, sizeof(flag)) != 0) { + FF_DEBUG("Failed to set SO_NOSIGPIPE: %s", strerror(errno)); + } else { + FF_DEBUG("Successfully set SO_NOSIGPIPE"); + } +#endif + if (state->timeout > 0) { FF_DEBUG("Setting connection timeout: %u ms", state->timeout); [[maybe_unused]] uint32_t sec = state->timeout / 1000; @@ -298,8 +325,8 @@ error: return ret; } -const char* ffNetworkingSendHttpRequest(FFNetworkingState* state, const char* host, const char* path, const char* headers) { - FF_DEBUG("Preparing to send HTTP request: host=%s, path=%s", host, path); +const char* ffNetworkingSendHttpRequest(FFNetworkingState* state, const char* host, uint16_t port, const char* path, const char* headers) { + FF_DEBUG("Preparing to send HTTP request: host=%s, port=%u, path=%s", host, port, path); if (state->compression) { FF_DEBUG("Compression enabled, checking if zlib is available"); @@ -321,7 +348,7 @@ const char* ffNetworkingSendHttpRequest(FFNetworkingState* state, const char* ho FF_DEBUG("Compression disabled"); } - const char* initResult = initNetworkingState(state, host, path, headers); + const char* initResult = initNetworkingState(state, host, port, path, headers); if (initResult != nullptr) { FF_DEBUG("Initialization failed: %s", initResult); return initResult; @@ -375,9 +402,20 @@ const char* ffNetworkingRecvHttpResponse(FFNetworkingState* state, FFstrbuf* buf } // Set larger initial receive buffer instead of small repeated receives - int rcvbuf = 65536; // 64KB + int rcvbuf = 64 * 1024; setsockopt(state->sockfd, SOL_SOCKET, SO_RCVBUF, &rcvbuf, sizeof(rcvbuf)); + // The timeout has to be enforced by the socket itself on every platform: the poll() below + // only reports readability once, so a server that sends a partial response and then keeps + // the connection open would otherwise block this loop forever. + if (timeout > 0) { + FF_DEBUG("Setting receive timeout: %u ms", timeout); + struct timeval timev; + timev.tv_sec = timeout / 1000; + timev.tv_usec = (typeof(timev.tv_usec)) ((timeout % 1000) * 1000); // milliseconds to microseconds + setsockopt(state->sockfd, SOL_SOCKET, SO_RCVTIMEO, &timev, sizeof(timev)); + } + #ifdef __APPLE__ // poll for the socket to be readable. // Because of the non-blocking connectx() call, the connection might not be established yet @@ -401,14 +439,6 @@ const char* ffNetworkingRecvHttpResponse(FFNetworkingState* state, FFstrbuf* buf } } FF_DEBUG("Socket is readable, proceeding to receive data"); -#else - if (timeout > 0) { - FF_DEBUG("Setting receive timeout: %u ms", timeout); - struct timeval timev; - timev.tv_sec = timeout / 1000; - timev.tv_usec = (typeof(timev.tv_usec)) ((timeout % 1000) * 1000); // milliseconds to microseconds - setsockopt(state->sockfd, SOL_SOCKET, SO_RCVTIMEO, &timev, sizeof(timev)); - } #endif if (shutdown(state->sockfd, SHUT_WR) == -1) { @@ -420,16 +450,46 @@ const char* ffNetworkingRecvHttpResponse(FFNetworkingState* state, FFstrbuf* buf [[maybe_unused]] int recvCount = 0; uint32_t contentLength = 0; uint32_t headerEnd = 0; + bool chunked = false; - do { - FF_DEBUG("Data reception loop #%d, current buffer size: %u, available space: %u", + // Runs until the response is framed; the buffer is grown on demand at the top + for (;;) { + if (ffStrbufGetFree(buffer) == 0) { + // `Content-Length` may be absent (e.g. chunked responses). Grow the buffer + // on demand instead of silently truncating the response. + if (buffer->allocated >= FF_NETWORKING_MAX_RESPONSE_SIZE) { + FF_DEBUG("Response is too large: %u bytes, aborting", buffer->allocated); + close(state->sockfd); + state->sockfd = -1; + return "Response too large"; + } + FF_DEBUG("Receive buffer is full, extending it"); + ffStrbufEnsureFreeNoCheck(buffer, buffer->allocated); + } + + // When the remaining length is known, ask for exactly that much. MSG_WAITALL then + // returns as soon as the response is complete, instead of waiting for the server + // to close the connection. + // Without a Content-Length the requested length is just "whatever fits", so + // waiting for all of it would block until the server closes -- and the framing + // checks below would never get a chance to run. Read whatever has arrived instead + // and let those checks decide when the response is complete. + uint32_t want = ffStrbufGetFree(buffer); + int recvFlags = 0; + if (contentLength > 0 && headerEnd > 0) { + uint32_t remaining = headerEnd + 4 + contentLength - buffer->length; + if (remaining < want) { + want = remaining; + } + recvFlags = MSG_WAITALL; + } + + FF_DEBUG("Data reception loop #%d, current buffer size: %u, requesting %u bytes", ++recvCount, buffer->length, - ffStrbufGetFree(buffer)); + want); - // We set `Connection: close`, so the server will close the connection when done. - // Thus we can use MSG_WAITALL to wait until the buffer is full or the connection is closed. - ssize_t received = recv(state->sockfd, buffer->chars + buffer->length, ffStrbufGetFree(buffer), MSG_WAITALL); + ssize_t received = recv(state->sockfd, buffer->chars + buffer->length, want, recvFlags); if (received <= 0) { if (received == 0) { @@ -453,11 +513,12 @@ const char* ffNetworkingRecvHttpResponse(FFNetworkingState* state, FFstrbuf* buf FF_DEBUG("Found HTTP header end marker, position: %u", headerEnd); // Check for Content-Length header to pre-allocate enough memory - const char* clHeader = strcasestr(buffer->chars, "Content-Length:"); + uint32_t valueLen = 0; + const char* clHeader = ffNetworkingFindHeader(buffer->chars, headerEnd, "Content-Length:", &valueLen); if (clHeader) { - contentLength = (uint32_t) strtoul(clHeader + 15, nullptr, 10); + contentLength = (uint32_t) strtoul(clHeader, nullptr, 10); if (contentLength > 0) { - if (contentLength > 1024 * 1024) { // 1MB limit to prevent excessive memory allocation and potential attacks + if (contentLength > FF_NETWORKING_MAX_RESPONSE_SIZE) { // 1MB limit to prevent excessive memory allocation and potential attacks FF_DEBUG("Content-Length is too large: %u bytes, aborting", contentLength); close(state->sockfd); state->sockfd = -1; @@ -470,9 +531,47 @@ const char* ffNetworkingRecvHttpResponse(FFNetworkingState* state, FFstrbuf* buf FF_DEBUG("Extended receive buffer to %u bytes", buffer->allocated); } } + + // A chunked response has no Content-Length; it is framed by a last-chunk + const char* teHeader = ffNetworkingFindHeader(buffer->chars, headerEnd, "Transfer-Encoding:", &valueLen); + if (teHeader != nullptr) { + switch (ffNetworkingParseTransferEncoding(teHeader, valueLen)) { + case FF_NETWORKING_TE_CHUNKED: + FF_DEBUG("Detected chunked transfer encoding"); + chunked = true; + break; + case FF_NETWORKING_TE_UNSUPPORTED: + // The framing of e.g. `gzip, chunked` is unreadable and the payload + // would stay encoded, so fail instead of returning garbage + FF_DEBUG("Unsupported Transfer-Encoding: %.*s", (int) valueLen, teHeader); + close(state->sockfd); + state->sockfd = -1; + return "Unsupported Transfer-Encoding"; + default: + break; + } + } } } - } while (ffStrbufGetFree(buffer) > 0); + + // Stop as soon as the response is framed, rather than waiting for the FIN + if (chunked) { + uint32_t consumed = 0; + int complete = ffNetworkingChunkedComplete(buffer->chars + headerEnd + 4, buffer->length - headerEnd - 4, &consumed); + if (complete < 0) { + FF_DEBUG("Malformed chunked body"); + close(state->sockfd); + state->sockfd = -1; + return "Malformed chunked body"; + } + if (complete > 0) { + FF_DEBUG("Chunked body complete, %u bytes of encoded body", consumed); + break; + } + } else if (contentLength > 0 && buffer->length >= headerEnd + 4 + contentLength) { + break; + } + } FF_DEBUG("Closing socket: fd=%d", state->sockfd); close(state->sockfd); @@ -488,6 +587,10 @@ const char* ffNetworkingRecvHttpResponse(FFNetworkingState* state, FFstrbuf* buf return "No HTTP header end found"; } + if (chunked && !ffNetworkingDecodeChunked(buffer, headerEnd)) { + return "Failed to decode chunked response"; + } + if (!ffStrbufStartsWithS(buffer, "HTTP/1.0 200 OK\r\n") && !ffStrbufStartsWithS(buffer, "HTTP/1.1 200 OK\r\n")) { FF_DEBUG("Invalid response: %.40s...", buffer->chars); return "Invalid response"; diff --git a/src/common/impl/networking_windows.c b/src/common/impl/networking_windows.c index 409edff7e..f5ed6c400 100644 --- a/src/common/impl/networking_windows.c +++ b/src/common/impl/networking_windows.c @@ -9,6 +9,9 @@ static LPFN_CONNECTEX ConnectEx; +// Upper bound of a single HTTP response, guarding against excessive memory allocation +#define FF_NETWORKING_MAX_RESPONSE_SIZE (1024u * 1024u) + static const char* initWsaData(WSADATA* wsaData) { FF_DEBUG("Initializing WinSock"); if (WSAStartup(MAKEWORD(2, 2), wsaData) != 0) { @@ -45,8 +48,8 @@ static const char* initWsaData(WSADATA* wsaData) { return nullptr; } -const char* ffNetworkingSendHttpRequest(FFNetworkingState* state, const char* host, const char* path, const char* headers) { - FF_DEBUG("Preparing to send HTTP request: host=%s, path=%s", host, path); +const char* ffNetworkingSendHttpRequest(FFNetworkingState* state, const char* host, uint16_t port, const char* path, const char* headers) { + FF_DEBUG("Preparing to send HTTP request: host=%s, port=%u, path=%s", host, port, path); if (state->compression) { #ifdef FF_HAVE_ZLIB @@ -92,8 +95,11 @@ const char* ffNetworkingSendHttpRequest(FFNetworkingState* state, const char* ho return "Failed to convert host to wide string"; } - FF_DEBUG("Resolving address: %s (%s)", host, state->ipv6 ? "IPv6" : "IPv4"); - if (GetAddrInfoW(hostW, L"80", &hints, &addr) != 0) { + wchar_t portW[6]; + _itow(port, portW, 10); + + FF_DEBUG("Resolving address: %s:%u (%s)", host, port, state->ipv6 ? "IPv6" : "IPv4"); + if (GetAddrInfoW(hostW, portW, &hints, &addr) != 0) { FF_DEBUG("GetAddrInfoW() failed"); return "GetAddrInfoW() failed"; } @@ -156,7 +162,19 @@ const char* ffNetworkingSendHttpRequest(FFNetworkingState* state, const char* ho ffStrbufAppendS(&state->command, "GET "); ffStrbufAppendS(&state->command, path); ffStrbufAppendS(&state->command, " HTTP/1.0\r\nHost: "); - ffStrbufAppendS(&state->command, host); + if (strchr(host, ':') != nullptr) { + // An IPv6 literal has to be bracketed in the Host header (RFC 9110 7.2), while + // GetAddrInfoW() wants it bare + ffStrbufAppendC(&state->command, '['); + ffStrbufAppendS(&state->command, host); + ffStrbufAppendC(&state->command, ']'); + } else { + ffStrbufAppendS(&state->command, host); + } + // The Host header carries the port whenever it is not the default one (RFC 9110 7.2) + if (port != 80) { + ffStrbufAppendF(&state->command, ":%u", port); + } ffStrbufAppendS(&state->command, "\r\nConnection: close\r\n"); // Explicitly request connection closure // Add compression support if enabled @@ -266,8 +284,23 @@ const char* ffNetworkingRecvHttpResponse(FFNetworkingState* state, FFstrbuf* buf [[maybe_unused]] int recvCount = 0; uint32_t contentLength = 0; uint32_t headerEnd = 0; + bool chunked = false; + + // Runs until the response is framed; the buffer is grown on demand at the top + for (;;) { + if (ffStrbufGetFree(buffer) == 0) { + // `Content-Length` may be absent (e.g. chunked responses). Grow the buffer + // on demand instead of silently truncating the response. + if (buffer->allocated >= FF_NETWORKING_MAX_RESPONSE_SIZE) { + FF_DEBUG("Response is too large: %u bytes, aborting", buffer->allocated); + closesocket(state->sockfd); + state->sockfd = INVALID_SOCKET; + return "Response too large"; + } + FF_DEBUG("Receive buffer is full, extending it"); + ffStrbufEnsureFreeNoCheck(buffer, buffer->allocated); + } - do { FF_DEBUG("Data reception loop #%d, current buffer size: %u, available space: %u", ++recvCount, buffer->length, @@ -306,11 +339,12 @@ const char* ffNetworkingRecvHttpResponse(FFNetworkingState* state, FFstrbuf* buf FF_DEBUG("Found HTTP header end marker, position: %u", headerEnd); // Check for Content-Length header to pre-allocate enough memory - const char* clHeader = strcasestr(buffer->chars, "Content-Length:"); + uint32_t valueLen = 0; + const char* clHeader = ffNetworkingFindHeader(buffer->chars, headerEnd, "Content-Length:", &valueLen); if (clHeader) { - contentLength = (uint32_t) strtoul(clHeader + 15, nullptr, 10); + contentLength = (uint32_t) strtoul(clHeader, nullptr, 10); if (contentLength > 0) { - if (contentLength > 1024 * 1024) { // 1MB limit to prevent excessive memory allocation and potential attacks + if (contentLength > FF_NETWORKING_MAX_RESPONSE_SIZE) { // 1MB limit to prevent excessive memory allocation and potential attacks FF_DEBUG("Content-Length is too large: %u bytes, aborting", contentLength); closesocket(state->sockfd); state->sockfd = INVALID_SOCKET; @@ -323,9 +357,47 @@ const char* ffNetworkingRecvHttpResponse(FFNetworkingState* state, FFstrbuf* buf FF_DEBUG("Extended receive buffer to %u bytes", buffer->allocated); } } + + // A chunked response has no Content-Length; it is framed by a last-chunk + const char* teHeader = ffNetworkingFindHeader(buffer->chars, headerEnd, "Transfer-Encoding:", &valueLen); + if (teHeader != nullptr) { + switch (ffNetworkingParseTransferEncoding(teHeader, valueLen)) { + case FF_NETWORKING_TE_CHUNKED: + FF_DEBUG("Detected chunked transfer encoding"); + chunked = true; + break; + case FF_NETWORKING_TE_UNSUPPORTED: + // The framing of e.g. `gzip, chunked` is unreadable and the payload + // would stay encoded, so fail instead of returning garbage + FF_DEBUG("Unsupported Transfer-Encoding: %.*s", (int) valueLen, teHeader); + closesocket(state->sockfd); + state->sockfd = INVALID_SOCKET; + return "Unsupported Transfer-Encoding"; + default: + break; + } + } } } - } while (ffStrbufGetFree(buffer) > 0); + + // Stop as soon as the response is framed, rather than waiting for the FIN + if (chunked) { + uint32_t consumed = 0; + int complete = ffNetworkingChunkedComplete(buffer->chars + headerEnd + 4, buffer->length - headerEnd - 4, &consumed); + if (complete < 0) { + FF_DEBUG("Malformed chunked body"); + closesocket(state->sockfd); + state->sockfd = INVALID_SOCKET; + return "Malformed chunked body"; + } + if (complete > 0) { + FF_DEBUG("Chunked body complete, %u bytes of encoded body", consumed); + break; + } + } else if (contentLength > 0 && buffer->length >= headerEnd + 4 + contentLength) { + break; + } + } FF_DEBUG("Closing socket: fd=%u", (unsigned) state->sockfd); closesocket(state->sockfd); @@ -341,6 +413,10 @@ const char* ffNetworkingRecvHttpResponse(FFNetworkingState* state, FFstrbuf* buf return "No HTTP header end found"; } + if (chunked && !ffNetworkingDecodeChunked(buffer, headerEnd)) { + return "Failed to decode chunked response"; + } + if (!ffStrbufStartsWithS(buffer, "HTTP/1.0 200 OK\r\n") && !ffStrbufStartsWithS(buffer, "HTTP/1.1 200 OK\r\n")) { FF_DEBUG("Invalid response: %.40s...", buffer->chars); return "Invalid response"; diff --git a/src/common/networking.h b/src/common/networking.h index 2554e2231..f7dbba422 100644 --- a/src/common/networking.h +++ b/src/common/networking.h @@ -29,9 +29,41 @@ typedef struct FFNetworkingState { bool tfo; // if true, TCP Fast Open will be attempted first, and fallback to traditional connection if it fails } FFNetworkingState; -const char* ffNetworkingSendHttpRequest(FFNetworkingState* state, const char* host, const char* path, const char* headers); +const char* ffNetworkingSendHttpRequest(FFNetworkingState* state, const char* host, uint16_t port, const char* path, const char* headers); const char* ffNetworkingRecvHttpResponse(FFNetworkingState* state, FFstrbuf* buffer); +// Case-insensitive header lookup restricted to the header block [0, headerEnd). +// Restricting the range matters because the body may already share the same buffer. +// Returns a pointer to the first character of the value; `valueLen` receives its +// length excluding the terminating CRLF. Returns nullptr when the header is absent. +const char* ffNetworkingFindHeader(const char* headers, uint32_t headerEnd, const char* name, uint32_t* valueLen); + +// Checks whether a `Transfer-Encoding: chunked` body has been received in full, so that +// framing does not have to rely on the server closing the connection. +// Returns 1 when complete (`consumed` receives the body length including trailers), +// 0 when more data is needed, and -1 when the body is malformed. +// The caller must not wait for a fixed amount of data (e.g. `MSG_WAITALL`) while the +// response length is still unknown, otherwise this check never gets to run. +int ffNetworkingChunkedComplete(const char* body, uint32_t bodyLen, uint32_t* consumed); + +// Decodes a `Transfer-Encoding: chunked` body in place and rewrites the response with a +// `Content-Length` header in place of `Transfer-Encoding`. +bool ffNetworkingDecodeChunked(FFstrbuf* buffer, uint32_t headerEnd); + +// Result of parsing a `Transfer-Encoding` header value +typedef enum FFNetworkingTransferEncoding { + FF_NETWORKING_TE_NONE, // the value holds no coding at all + FF_NETWORKING_TE_CHUNKED, // exactly `chunked`, the only framing this client decodes + FF_NETWORKING_TE_UNSUPPORTED, // another coding or a chain of them, e.g. `gzip, chunked` +} FFNetworkingTransferEncoding; + +// Parses a `Transfer-Encoding` header value. Codings are applied in the order they are +// listed, so `chunked` has to be the last one for the framing to be readable at all -- +// and any other coding (e.g. `gzip, chunked`) leaves the payload encoded, which this +// client cannot decode. Only a lone `chunked` is accepted; everything else is reported +// as unsupported so that the caller fails the response instead of returning garbage. +FFNetworkingTransferEncoding ffNetworkingParseTransferEncoding(const char* value, uint32_t valueLen); + #ifdef FF_HAVE_ZLIB const char* ffNetworkingLoadZlibLibrary(void); bool ffNetworkingDecompressGzip(FFstrbuf* buffer, char* headerEnd); diff --git a/src/detection/publicip/publicip.c b/src/detection/publicip/publicip.c index c0e8afb47..90efa12d2 100644 --- a/src/detection/publicip/publicip.c +++ b/src/detection/publicip/publicip.c @@ -5,6 +5,20 @@ static FFNetworkingState states[2]; static const char* statuses[2] = { FF_UNINITIALIZED, FF_UNINITIALIZED }; +// Reads the port that follows the colon at `colonIndex`. The port has to be the last thing in +// `host`, so this must run before any part of the host is trimmed off. +static uint16_t ffPublicIpParseUrlPort(const FFstrbuf* host, uint32_t colonIndex) { + const char* portStr = host->chars + colonIndex + 1; + char* portEnd = nullptr; + unsigned long portValue = strtoul(portStr, &portEnd, 10); + if (portEnd == portStr || *portEnd != '\0' || portValue == 0 || portValue > 65535) { + fputs("Error: invalid port in the PublicIp module URL\n", stderr); + exit(1); + } + + return (uint16_t) portValue; +} + void ffPreparePublicIp(FFPublicIPOptions* options) { FFNetworkingState* state = &states[options->ipv6]; const char** status = &statuses[options->ipv6]; @@ -19,7 +33,7 @@ void ffPreparePublicIp(FFPublicIPOptions* options) { if (options->url.length == 0) { state->compression = true; state->tfo = true; - *status = ffNetworkingSendHttpRequest(state, options->ipv6 ? "v6.ipinfo.io" : "ipinfo.io", "/json", nullptr); + *status = ffNetworkingSendHttpRequest(state, options->ipv6 ? "v6.ipinfo.io" : "ipinfo.io", 80, "/json", nullptr); } else { FF_STRBUF_AUTO_DESTROY host = ffStrbufCreateCopy(&options->url); uint32_t hostStartIndex = ffStrbufFirstIndexS(&host, "://"); @@ -38,7 +52,39 @@ void ffPreparePublicIp(FFPublicIPOptions* options) { ffStrbufSubstrBefore(&host, pathStartIndex); } - *status = ffNetworkingSendHttpRequest(state, host.chars, path.length == 0 ? "/" : path.chars, nullptr); + // An optional `:port` must be split off the host, otherwise getaddrinfo() is asked to + // resolve a host name that still carries the port. Only a single colon can separate a + // port: a bare IPv6 literal (`::1`) holds several of them and has no port at all, + // which is why a bracketed literal (`[::1]:8080`) is the only unambiguous spelling. + uint16_t port = 0; + if (ffStrbufStartsWithC(&host, '[')) { + uint32_t bracketEnd = ffStrbufFirstIndexC(&host, ']'); + if (bracketEnd == host.length) { + fputs("Error: unmatched '[' in the PublicIp module URL\n", stderr); + exit(1); + } + + if (bracketEnd + 1 < host.length) { + if (host.chars[bracketEnd + 1] != ':') { + fputs("Error: unexpected characters after the IPv6 literal in the PublicIp module URL\n", stderr); + exit(1); + } + // Read the port while the string is still intact: trimming the brackets first + // would invalidate the index it was found at. + port = ffPublicIpParseUrlPort(&host, bracketEnd + 1); + } + + ffStrbufSubstrBefore(&host, bracketEnd); + ffStrbufSubstrAfter(&host, 0); // drop the leading '[' + } else { + uint32_t firstColon = ffStrbufFirstIndexC(&host, ':'); + if (firstColon < host.length && firstColon == ffStrbufLastIndexC(&host, ':')) { + port = ffPublicIpParseUrlPort(&host, firstColon); + ffStrbufSubstrBefore(&host, firstColon); + } + } + + *status = ffNetworkingSendHttpRequest(state, host.chars, port ?: 80, path.length == 0 ? "/" : path.chars, nullptr); } } diff --git a/src/detection/weather/weather.c b/src/detection/weather/weather.c index 8a70a966a..2899456f1 100644 --- a/src/detection/weather/weather.c +++ b/src/detection/weather/weather.c @@ -29,7 +29,7 @@ void ffPrepareWeather(FFWeatherOptions* options) { default: break; } - status = ffNetworkingSendHttpRequest(&state, "wttr.in", path.chars, "User-Agent: curl/0.0.0\r\n"); + status = ffNetworkingSendHttpRequest(&state, "wttr.in", 80, path.chars, "User-Agent: curl/0.0.0\r\n"); } const char* ffDetectWeather(FFWeatherOptions* options, FFstrbuf* result) { diff --git a/tests/networking.c b/tests/networking.c new file mode 100644 index 000000000..18f0fc927 --- /dev/null +++ b/tests/networking.c @@ -0,0 +1,86 @@ +#include "common/networking.h" + +#include +#include +#include + +static void verify(bool expression, const char* expressionStr, int lineNo) { + if (expression) { + return; + } + + fprintf(stderr, "[%d] %s\n", lineNo, expressionStr); + exit(1); +} + +#define VERIFY(expression) verify((expression), #expression, __LINE__) + +static FFNetworkingTransferEncoding parse(const char* value) { + return ffNetworkingParseTransferEncoding(value, (uint32_t) strlen(value)); +} + +int main(void) { + { + VERIFY(parse("chunked") == FF_NETWORKING_TE_CHUNKED); + VERIFY(parse("Chunked") == FF_NETWORKING_TE_CHUNKED); + VERIFY(parse(" chunked ") == FF_NETWORKING_TE_CHUNKED); + VERIFY(parse("chunked\t") == FF_NETWORKING_TE_CHUNKED); + + // A coding chain leaves the payload encoded, so only a lone `chunked` is decodable + VERIFY(parse("gzip, chunked") == FF_NETWORKING_TE_UNSUPPORTED); + VERIFY(parse("gzip,chunked") == FF_NETWORKING_TE_UNSUPPORTED); + VERIFY(parse("chunked, gzip") == FF_NETWORKING_TE_UNSUPPORTED); + VERIFY(parse("gzip") == FF_NETWORKING_TE_UNSUPPORTED); + VERIFY(parse("x-chunked") == FF_NETWORKING_TE_UNSUPPORTED); + VERIFY(parse("chunkedx") == FF_NETWORKING_TE_UNSUPPORTED); + + VERIFY(parse("") == FF_NETWORKING_TE_NONE); + VERIFY(parse(" , ") == FF_NETWORKING_TE_NONE); + } + + { + // The lookup is restricted to the header block, so a body that happens to contain a + // `Transfer-Encoding:` line can never be mistaken for a header + const char* response = "HTTP/1.1 200 OK\r\nContent-Length: 4\r\n\r\nTransfer-Encoding: chunked\r\n"; + uint32_t headerEnd = (uint32_t) (strstr(response, "\r\n\r\n") - response); + uint32_t valueLen = 0; + + VERIFY(ffNetworkingFindHeader(response, headerEnd, "Transfer-Encoding:", &valueLen) == nullptr); + + const char* clHeader = ffNetworkingFindHeader(response, headerEnd, "Content-Length:", &valueLen); + VERIFY(clHeader != nullptr && valueLen == 1 && *clHeader == '4'); + } + + { + uint32_t consumed = 0; + const char* complete = "5\r\nhello\r\n0\r\n\r\n"; + VERIFY(ffNetworkingChunkedComplete(complete, (uint32_t) strlen(complete), &consumed) == 1); + VERIFY(consumed == strlen(complete)); + + const char* withTrailer = "5\r\nhello\r\n0\r\nX-Foo: bar\r\n\r\n"; + VERIFY(ffNetworkingChunkedComplete(withTrailer, (uint32_t) strlen(withTrailer), &consumed) == 1); + VERIFY(consumed == strlen(withTrailer)); + + const char* partial = "5\r\nhel"; + VERIFY(ffNetworkingChunkedComplete(partial, (uint32_t) strlen(partial), &consumed) == 0); + + const char* malformed = "5\r\nhelloXX0\r\n\r\n"; + VERIFY(ffNetworkingChunkedComplete(malformed, (uint32_t) strlen(malformed), &consumed) == -1); + } + + { + // Decoding replaces the framing header with a matching Content-Length + FFstrbuf response = ffStrbufCreateS("HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n5\r\nhello\r\n0\r\n\r\n"); + uint32_t headerEnd = (uint32_t) (strstr(response.chars, "\r\n\r\n") - response.chars); + + VERIFY(ffNetworkingDecodeChunked(&response, headerEnd)); + VERIFY(ffStrbufContainS(&response, "Content-Length: 5")); + VERIFY(!ffStrbufContainS(&response, "Transfer-Encoding")); + VERIFY(ffStrbufEndsWithS(&response, "hello")); + + ffStrbufDestroy(&response); + } + + puts("All networking tests passed!"); + return 0; +}