Networking: supports chunked response & custom port number

src/common/impl/networking_common.c

@@ -190,3 +190,212 @@ bool ffNetworkingDecompressGzip(FFstrbuf* buffer, char* headerEnd) {
     return true;
 }
 #endif // FF_HAVE_ZLIB
+
+const char* ffNetworkingFindHeader(const char* headers, uint32_t headerEnd, const char* name, uint32_t* valueLen) {
+    assert(headers != nullptr && valueLen != nullptr);
+
+    uint32_t nameLen = (uint32_t) strlen(name);
+    uint32_t pos = 0;
+
+    while (pos < headerEnd) {
+        uint32_t eol = pos;
+        while (eol < headerEnd && headers[eol] != '\n') {
+            ++eol;
+        }
+        uint32_t lineEnd = (eol > pos && headers[eol - 1] == '\r') ? (eol - 1) : eol;
+
+        // Only match at the beginning of a line, so that a value can never be mistaken
+        // for a field name (obs-fold continuation lines included)
+        if (lineEnd - pos >= nameLen && strncasecmp(headers + pos, name, nameLen) == 0) {
+            uint32_t valueStart = pos + nameLen;
+            while (valueStart < lineEnd && (headers[valueStart] == ' ' || headers[valueStart] == '\t')) {
+                ++valueStart;
+            }
+            *valueLen = lineEnd - valueStart;
+            return headers + valueStart;
+        }
+
+        if (eol >= headerEnd) {
+            break;
+        }
+        pos = eol + 1;
+    }
+
+    return nullptr;
+}
+
+FFNetworkingTransferEncoding ffNetworkingParseTransferEncoding(const char* value, uint32_t valueLen) {
+    assert(value != nullptr);
+
+    // The value is a comma-separated list of transfer codings (RFC 9112 6.1)
+    uint32_t codingCount = 0;
+    const char* coding = nullptr;
+    uint32_t codingLen = 0;
+
+    for (uint32_t i = 0, start = 0; i <= valueLen; ++i) {
+        if (i < valueLen && value[i] != ',') {
+            continue;
+        }
+
+        // trim the optional whitespace around the coding
+        uint32_t from = start;
+        uint32_t to = i;
+        while (from < to && (value[from] == ' ' || value[from] == '\t')) {
+            ++from;
+        }
+        while (to > from && (value[to - 1] == ' ' || value[to - 1] == '\t')) {
+            --to;
+        }
+
+        if (to > from) {
+            ++codingCount;
+            coding = value + from;
+            codingLen = to - from;
+        }
+        start = i + 1;
+    }
+
+    if (codingCount == 0) {
+        return FF_NETWORKING_TE_NONE;
+    }
+
+    if (codingCount == 1 && codingLen == 7 && strncasecmp(coding, "chunked", 7) == 0) {
+        return FF_NETWORKING_TE_CHUNKED;
+    }
+
+    return FF_NETWORKING_TE_UNSUPPORTED;
+}
+
+int ffNetworkingChunkedComplete(const char* body, uint32_t bodyLen, uint32_t* consumed) {
+    assert(body != nullptr && consumed != nullptr);
+
+    uint32_t pos = 0;
+
+    for (;;) {
+        // chunk-size [ chunk-ext ] CRLF
+        uint32_t eol = pos;
+        while (eol < bodyLen && body[eol] != '\n') {
+            ++eol;
+        }
+        if (eol >= bodyLen) {
+            return 0; // the chunk-size line is not complete yet
+        }
+
+        if (eol == pos || !isxdigit((unsigned char) body[pos])) {
+            return -1;
+        }
+
+        char* stop = nullptr;
+        unsigned long size = strtoul(body + pos, &stop, 16);
+        if (stop == body + pos) {
+            return -1;
+        }
+        pos = eol + 1; // skips the chunk extension, which ends at the CRLF
+
+        if (size == 0) {
+            // last-chunk, followed by an optional trailer section and an empty line
+            uint32_t p = pos;
+            while (p < bodyLen) {
+                uint32_t e = p;
+                while (e < bodyLen && body[e] != '\n') {
+                    ++e;
+                }
+                if (e >= bodyLen) {
+                    return 0;
+                }
+                if (e == p || (e == p + 1 && body[p] == '\r')) {
+                    *consumed = e + 1;
+                    return 1;
+                }
+                p = e + 1;
+            }
+            return 0;
+        }
+
+        if (size > (unsigned long) (bodyLen - pos)) {
+            return 0; // the chunk data is not complete yet
+        }
+        pos += (uint32_t) size;
+
+        if (bodyLen - pos < 2) {
+            return 0;
+        }
+        if (body[pos] != '\r' || body[pos + 1] != '\n') {
+            return -1;
+        }
+        pos += 2;
+    }
+}
+
+// Keeps the status line and every header except `dropHeader` and `Content-Length`,
+// then emits a `Content-Length` matching the (already decoded) body.
+// `body` may point into `buffer->chars`; it is copied before `buffer` is released.
+static void rebuildResponse(FFstrbuf* buffer, uint32_t headerEnd, const char* body, uint32_t bodyLen, const char* dropHeader) {
+    FF_STRBUF_AUTO_DESTROY newBuffer = ffStrbufCreateA(headerEnd + bodyLen + 64);
+
+    uint32_t pos = 0;
+    while (pos < headerEnd) {
+        uint32_t eol = pos;
+        while (eol < headerEnd && buffer->chars[eol] != '\n') {
+            ++eol;
+        }
+        uint32_t lineLen = (eol < headerEnd) ? (eol - pos + 1) : (headerEnd - pos);
+
+        if (!ffStrStartsWithIgnCase(buffer->chars + pos, "Content-Length:") &&
+            !ffStrStartsWithIgnCase(buffer->chars + pos, dropHeader)) {
+            ffStrbufAppendNS(&newBuffer, lineLen, buffer->chars + pos);
+        }
+
+        pos += lineLen;
+    }
+
+    ffStrbufAppendF(&newBuffer, "Content-Length: %u\r\n\r\n", bodyLen);
+    ffStrbufAppendNS(&newBuffer, bodyLen, body);
+
+    ffStrbufDestroy(buffer);
+    ffStrbufInitMove(buffer, &newBuffer);
+}
+
+bool ffNetworkingDecodeChunked(FFstrbuf* buffer, uint32_t headerEnd) {
+    assert(buffer->allocated > 0);
+
+    // The header block is terminated by CR LF CR LF
+    if (headerEnd + 4 > buffer->length) {
+        return false;
+    }
+
+    char* body = buffer->chars + headerEnd + 4;
+    uint32_t bodyLen = buffer->length - headerEnd - 4;
+
+    uint32_t consumed = 0;
+    if (ffNetworkingChunkedComplete(body, bodyLen, &consumed) != 1) {
+        FF_DEBUG("Incomplete or malformed chunked body");
+        return false;
+    }
+
+    // Decoding in place is safe: the encoded form is never shorter than the payload
+    char* out = body;
+    uint32_t outLen = 0;
+    uint32_t pos = 0;
+
+    while (pos < bodyLen) {
+        uint32_t eol = pos;
+        while (eol < bodyLen && body[eol] != '\n') {
+            ++eol;
+        }
+
+        unsigned long size = strtoul(body + pos, nullptr, 16);
+        pos = eol + 1;
+        if (size == 0) {
+            break; // last-chunk; trailers are dropped
+        }
+
+        memmove(out + outLen, body + pos, size);
+        outLen += (uint32_t) size;
+        pos += (uint32_t) size + 2; // trailing CRLF
+    }
+
+    FF_DEBUG("Decoded chunked body: %u bytes encoded, %u bytes decoded", bodyLen, outLen);
+    rebuildResponse(buffer, headerEnd, out, outLen, "Transfer-Encoding:");
+    return true;
+}

src/common/impl/networking_linux.c

@@ -16,6 +16,9 @@
 #include <errno.h>
 #include <fcntl.h>
 
+// Upper bound of a single HTTP response, guarding against excessive memory allocation
+#define FF_NETWORKING_MAX_RESPONSE_SIZE (1024u * 1024u)
+
 static const char* tryNonThreadingFastPath(FFNetworkingState* state) {
 #if defined(TCP_FASTOPEN) || __APPLE__
 
@@ -187,7 +190,7 @@ exit:
 FF_THREAD_ENTRY_DECL_WRAPPER(connectAndSend, FFNetworkingState*);
 
 // Parallel DNS resolution and socket creation
-static const char* initNetworkingState(FFNetworkingState* state, const char* host, const char* path, const char* headers) {
+static const char* initNetworkingState(FFNetworkingState* state, const char* host, uint16_t port, const char* path, const char* headers) {
     FF_DEBUG("Initializing network connection state: host=%s, path=%s", host, path);
 
     // Initialize command and host information
@@ -195,7 +198,19 @@ static const char* initNetworkingState(FFNetworkingState* state, const char* hos
     ffStrbufAppendS(&state->command, "GET ");
     ffStrbufAppendS(&state->command, path);
     ffStrbufAppendS(&state->command, " HTTP/1.0\r\nHost: ");
-    ffStrbufAppendS(&state->command, host);
+    if (strchr(host, ':') != nullptr) {
+        // An IPv6 literal has to be bracketed in the Host header (RFC 9110 7.2), while
+        // getaddrinfo() wants it bare
+        ffStrbufAppendC(&state->command, '[');
+        ffStrbufAppendS(&state->command, host);
+        ffStrbufAppendC(&state->command, ']');
+    } else {
+        ffStrbufAppendS(&state->command, host);
+    }
+    // The Host header carries the port whenever it is not the default one (RFC 9110 7.2)
+    if (port != 80) {
+        ffStrbufAppendF(&state->command, ":%u", port);
+    }
     ffStrbufAppendS(&state->command, "\r\nConnection: close\r\n"); // Explicitly tell the server we don't need to keep the connection
 
     // If compression needs to be enabled
@@ -220,10 +235,13 @@ static const char* initNetworkingState(FFNetworkingState* state, const char* hos
         .ai_flags = AI_NUMERICSERV
     };
 
-    FF_DEBUG("Resolving address: %s (%s)", host, state->ipv6 ? "IPv6" : "IPv4");
+    char portA[6];
+    snprintf(portA, sizeof(portA), "%u", port);
+
+    FF_DEBUG("Resolving address: %s:%u (%s)", host, port, state->ipv6 ? "IPv6" : "IPv4");
     // Use AI_NUMERICSERV flag to indicate the service is a numeric port, reducing parsing time
 
-    int gaiRes = getaddrinfo(host, "80", &hints, &state->addr);
+    int gaiRes = getaddrinfo(host, portA, &hints, &state->addr);
     if (gaiRes != 0) {
         FF_DEBUG("getaddrinfo() failed: %s (res=%d)", gai_strerror(gaiRes), gaiRes);
         ret = "getaddrinfo() failed";
@@ -259,6 +277,15 @@ static const char* initNetworkingState(FFNetworkingState* state, const char* hos
     }
 #endif
 
+#ifdef SO_NOSIGPIPE
+    // Prevent SIGPIPE when the server closes the connection during write
+    if (setsockopt(state->sockfd, SOL_SOCKET, SO_NOSIGPIPE, &flag, sizeof(flag)) != 0) {
+        FF_DEBUG("Failed to set SO_NOSIGPIPE: %s", strerror(errno));
+    } else {
+        FF_DEBUG("Successfully set SO_NOSIGPIPE");
+    }
+#endif
+
     if (state->timeout > 0) {
         FF_DEBUG("Setting connection timeout: %u ms", state->timeout);
         [[maybe_unused]] uint32_t sec = state->timeout / 1000;
@@ -298,8 +325,8 @@ error:
     return ret;
 }
 
-const char* ffNetworkingSendHttpRequest(FFNetworkingState* state, const char* host, const char* path, const char* headers) {
-    FF_DEBUG("Preparing to send HTTP request: host=%s, path=%s", host, path);
+const char* ffNetworkingSendHttpRequest(FFNetworkingState* state, const char* host, uint16_t port, const char* path, const char* headers) {
+    FF_DEBUG("Preparing to send HTTP request: host=%s, port=%u, path=%s", host, port, path);
 
     if (state->compression) {
         FF_DEBUG("Compression enabled, checking if zlib is available");
@@ -321,7 +348,7 @@ const char* ffNetworkingSendHttpRequest(FFNetworkingState* state, const char* ho
         FF_DEBUG("Compression disabled");
     }
 
-    const char* initResult = initNetworkingState(state, host, path, headers);
+    const char* initResult = initNetworkingState(state, host, port, path, headers);
     if (initResult != nullptr) {
         FF_DEBUG("Initialization failed: %s", initResult);
         return initResult;
@@ -375,9 +402,20 @@ const char* ffNetworkingRecvHttpResponse(FFNetworkingState* state, FFstrbuf* buf
     }
 
     // Set larger initial receive buffer instead of small repeated receives
-    int rcvbuf = 65536; // 64KB
+    int rcvbuf = 64 * 1024;
     setsockopt(state->sockfd, SOL_SOCKET, SO_RCVBUF, &rcvbuf, sizeof(rcvbuf));
 
+    // The timeout has to be enforced by the socket itself on every platform: the poll() below
+    // only reports readability once, so a server that sends a partial response and then keeps
+    // the connection open would otherwise block this loop forever.
+    if (timeout > 0) {
+        FF_DEBUG("Setting receive timeout: %u ms", timeout);
+        struct timeval timev;
+        timev.tv_sec = timeout / 1000;
+        timev.tv_usec = (typeof(timev.tv_usec)) ((timeout % 1000) * 1000); // milliseconds to microseconds
+        setsockopt(state->sockfd, SOL_SOCKET, SO_RCVTIMEO, &timev, sizeof(timev));
+    }
+
 #ifdef __APPLE__
     // poll for the socket to be readable.
     // Because of the non-blocking connectx() call, the connection might not be established yet
@@ -401,14 +439,6 @@ const char* ffNetworkingRecvHttpResponse(FFNetworkingState* state, FFstrbuf* buf
         }
     }
     FF_DEBUG("Socket is readable, proceeding to receive data");
-#else
-    if (timeout > 0) {
-        FF_DEBUG("Setting receive timeout: %u ms", timeout);
-        struct timeval timev;
-        timev.tv_sec = timeout / 1000;
-        timev.tv_usec = (typeof(timev.tv_usec)) ((timeout % 1000) * 1000); // milliseconds to microseconds
-        setsockopt(state->sockfd, SOL_SOCKET, SO_RCVTIMEO, &timev, sizeof(timev));
-    }
 #endif
 
     if (shutdown(state->sockfd, SHUT_WR) == -1) {
@@ -420,16 +450,46 @@ const char* ffNetworkingRecvHttpResponse(FFNetworkingState* state, FFstrbuf* buf
     [[maybe_unused]] int recvCount = 0;
     uint32_t contentLength = 0;
     uint32_t headerEnd = 0;
+    bool chunked = false;
 
-    do {
-        FF_DEBUG("Data reception loop #%d, current buffer size: %u, available space: %u",
+    // Runs until the response is framed; the buffer is grown on demand at the top
+    for (;;) {
+        if (ffStrbufGetFree(buffer) == 0) {
+            // `Content-Length` may be absent (e.g. chunked responses). Grow the buffer
+            // on demand instead of silently truncating the response.
+            if (buffer->allocated >= FF_NETWORKING_MAX_RESPONSE_SIZE) {
+                FF_DEBUG("Response is too large: %u bytes, aborting", buffer->allocated);
+                close(state->sockfd);
+                state->sockfd = -1;
+                return "Response too large";
+            }
+            FF_DEBUG("Receive buffer is full, extending it");
+            ffStrbufEnsureFreeNoCheck(buffer, buffer->allocated);
+        }
+
+        // When the remaining length is known, ask for exactly that much. MSG_WAITALL then
+        // returns as soon as the response is complete, instead of waiting for the server
+        // to close the connection.
+        // Without a Content-Length the requested length is just "whatever fits", so
+        // waiting for all of it would block until the server closes -- and the framing
+        // checks below would never get a chance to run. Read whatever has arrived instead
+        // and let those checks decide when the response is complete.
+        uint32_t want = ffStrbufGetFree(buffer);
+        int recvFlags = 0;
+        if (contentLength > 0 && headerEnd > 0) {
+            uint32_t remaining = headerEnd + 4 + contentLength - buffer->length;
+            if (remaining < want) {
+                want = remaining;
+            }
+            recvFlags = MSG_WAITALL;
+        }
+
+        FF_DEBUG("Data reception loop #%d, current buffer size: %u, requesting %u bytes",
             ++recvCount,
             buffer->length,
-            ffStrbufGetFree(buffer));
+            want);
 
-        // We set `Connection: close`, so the server will close the connection when done.
-        // Thus we can use MSG_WAITALL to wait until the buffer is full or the connection is closed.
-        ssize_t received = recv(state->sockfd, buffer->chars + buffer->length, ffStrbufGetFree(buffer), MSG_WAITALL);
+        ssize_t received = recv(state->sockfd, buffer->chars + buffer->length, want, recvFlags);
 
         if (received <= 0) {
             if (received == 0) {
@@ -453,11 +513,12 @@ const char* ffNetworkingRecvHttpResponse(FFNetworkingState* state, FFstrbuf* buf
                 FF_DEBUG("Found HTTP header end marker, position: %u", headerEnd);
 
                 // Check for Content-Length header to pre-allocate enough memory
-                const char* clHeader = strcasestr(buffer->chars, "Content-Length:");
+                uint32_t valueLen = 0;
+                const char* clHeader = ffNetworkingFindHeader(buffer->chars, headerEnd, "Content-Length:", &valueLen);
                 if (clHeader) {
-                    contentLength = (uint32_t) strtoul(clHeader + 15, nullptr, 10);
+                    contentLength = (uint32_t) strtoul(clHeader, nullptr, 10);
                     if (contentLength > 0) {
-                        if (contentLength > 1024 * 1024) { // 1MB limit to prevent excessive memory allocation and potential attacks
+                        if (contentLength > FF_NETWORKING_MAX_RESPONSE_SIZE) { // 1MB limit to prevent excessive memory allocation and potential attacks
                             FF_DEBUG("Content-Length is too large: %u bytes, aborting", contentLength);
                             close(state->sockfd);
                             state->sockfd = -1;
@@ -470,9 +531,47 @@ const char* ffNetworkingRecvHttpResponse(FFNetworkingState* state, FFstrbuf* buf
                         FF_DEBUG("Extended receive buffer to %u bytes", buffer->allocated);
                     }
                 }
+
+                // A chunked response has no Content-Length; it is framed by a last-chunk
+                const char* teHeader = ffNetworkingFindHeader(buffer->chars, headerEnd, "Transfer-Encoding:", &valueLen);
+                if (teHeader != nullptr) {
+                    switch (ffNetworkingParseTransferEncoding(teHeader, valueLen)) {
+                        case FF_NETWORKING_TE_CHUNKED:
+                            FF_DEBUG("Detected chunked transfer encoding");
+                            chunked = true;
+                            break;
+                        case FF_NETWORKING_TE_UNSUPPORTED:
+                            // The framing of e.g. `gzip, chunked` is unreadable and the payload
+                            // would stay encoded, so fail instead of returning garbage
+                            FF_DEBUG("Unsupported Transfer-Encoding: %.*s", (int) valueLen, teHeader);
+                            close(state->sockfd);
+                            state->sockfd = -1;
+                            return "Unsupported Transfer-Encoding";
+                        default:
+                            break;
+                    }
+                }
             }
         }
-    } while (ffStrbufGetFree(buffer) > 0);
+
+        // Stop as soon as the response is framed, rather than waiting for the FIN
+        if (chunked) {
+            uint32_t consumed = 0;
+            int complete = ffNetworkingChunkedComplete(buffer->chars + headerEnd + 4, buffer->length - headerEnd - 4, &consumed);
+            if (complete < 0) {
+                FF_DEBUG("Malformed chunked body");
+                close(state->sockfd);
+                state->sockfd = -1;
+                return "Malformed chunked body";
+            }
+            if (complete > 0) {
+                FF_DEBUG("Chunked body complete, %u bytes of encoded body", consumed);
+                break;
+            }
+        } else if (contentLength > 0 && buffer->length >= headerEnd + 4 + contentLength) {
+            break;
+        }
+    }
 
     FF_DEBUG("Closing socket: fd=%d", state->sockfd);
     close(state->sockfd);
@@ -488,6 +587,10 @@ const char* ffNetworkingRecvHttpResponse(FFNetworkingState* state, FFstrbuf* buf
         return "No HTTP header end found";
     }
 
+    if (chunked && !ffNetworkingDecodeChunked(buffer, headerEnd)) {
+        return "Failed to decode chunked response";
+    }
+
     if (!ffStrbufStartsWithS(buffer, "HTTP/1.0 200 OK\r\n") && !ffStrbufStartsWithS(buffer, "HTTP/1.1 200 OK\r\n")) {
         FF_DEBUG("Invalid response: %.40s...", buffer->chars);
         return "Invalid response";

src/common/impl/networking_windows.c

@@ -9,6 +9,9 @@
 
 static LPFN_CONNECTEX ConnectEx;
 
+// Upper bound of a single HTTP response, guarding against excessive memory allocation
+#define FF_NETWORKING_MAX_RESPONSE_SIZE (1024u * 1024u)
+
 static const char* initWsaData(WSADATA* wsaData) {
     FF_DEBUG("Initializing WinSock");
     if (WSAStartup(MAKEWORD(2, 2), wsaData) != 0) {
@@ -45,8 +48,8 @@ static const char* initWsaData(WSADATA* wsaData) {
     return nullptr;
 }
 
-const char* ffNetworkingSendHttpRequest(FFNetworkingState* state, const char* host, const char* path, const char* headers) {
-    FF_DEBUG("Preparing to send HTTP request: host=%s, path=%s", host, path);
+const char* ffNetworkingSendHttpRequest(FFNetworkingState* state, const char* host, uint16_t port, const char* path, const char* headers) {
+    FF_DEBUG("Preparing to send HTTP request: host=%s, port=%u, path=%s", host, port, path);
 
     if (state->compression) {
 #ifdef FF_HAVE_ZLIB
@@ -92,8 +95,11 @@ const char* ffNetworkingSendHttpRequest(FFNetworkingState* state, const char* ho
         return "Failed to convert host to wide string";
     }
 
-    FF_DEBUG("Resolving address: %s (%s)", host, state->ipv6 ? "IPv6" : "IPv4");
-    if (GetAddrInfoW(hostW, L"80", &hints, &addr) != 0) {
+    wchar_t portW[6];
+    _itow(port, portW, 10);
+
+    FF_DEBUG("Resolving address: %s:%u (%s)", host, port, state->ipv6 ? "IPv6" : "IPv4");
+    if (GetAddrInfoW(hostW, portW, &hints, &addr) != 0) {
         FF_DEBUG("GetAddrInfoW() failed");
         return "GetAddrInfoW() failed";
     }
@@ -156,7 +162,19 @@ const char* ffNetworkingSendHttpRequest(FFNetworkingState* state, const char* ho
     ffStrbufAppendS(&state->command, "GET ");
     ffStrbufAppendS(&state->command, path);
     ffStrbufAppendS(&state->command, " HTTP/1.0\r\nHost: ");
-    ffStrbufAppendS(&state->command, host);
+    if (strchr(host, ':') != nullptr) {
+        // An IPv6 literal has to be bracketed in the Host header (RFC 9110 7.2), while
+        // GetAddrInfoW() wants it bare
+        ffStrbufAppendC(&state->command, '[');
+        ffStrbufAppendS(&state->command, host);
+        ffStrbufAppendC(&state->command, ']');
+    } else {
+        ffStrbufAppendS(&state->command, host);
+    }
+    // The Host header carries the port whenever it is not the default one (RFC 9110 7.2)
+    if (port != 80) {
+        ffStrbufAppendF(&state->command, ":%u", port);
+    }
     ffStrbufAppendS(&state->command, "\r\nConnection: close\r\n"); // Explicitly request connection closure
 
     // Add compression support if enabled
@@ -266,8 +284,23 @@ const char* ffNetworkingRecvHttpResponse(FFNetworkingState* state, FFstrbuf* buf
     [[maybe_unused]] int recvCount = 0;
     uint32_t contentLength = 0;
     uint32_t headerEnd = 0;
+    bool chunked = false;
+
+    // Runs until the response is framed; the buffer is grown on demand at the top
+    for (;;) {
+        if (ffStrbufGetFree(buffer) == 0) {
+            // `Content-Length` may be absent (e.g. chunked responses). Grow the buffer
+            // on demand instead of silently truncating the response.
+            if (buffer->allocated >= FF_NETWORKING_MAX_RESPONSE_SIZE) {
+                FF_DEBUG("Response is too large: %u bytes, aborting", buffer->allocated);
+                closesocket(state->sockfd);
+                state->sockfd = INVALID_SOCKET;
+                return "Response too large";
+            }
+            FF_DEBUG("Receive buffer is full, extending it");
+            ffStrbufEnsureFreeNoCheck(buffer, buffer->allocated);
+        }
 
-    do {
         FF_DEBUG("Data reception loop #%d, current buffer size: %u, available space: %u",
             ++recvCount,
             buffer->length,
@@ -306,11 +339,12 @@ const char* ffNetworkingRecvHttpResponse(FFNetworkingState* state, FFstrbuf* buf
                 FF_DEBUG("Found HTTP header end marker, position: %u", headerEnd);
 
                 // Check for Content-Length header to pre-allocate enough memory
-                const char* clHeader = strcasestr(buffer->chars, "Content-Length:");
+                uint32_t valueLen = 0;
+                const char* clHeader = ffNetworkingFindHeader(buffer->chars, headerEnd, "Content-Length:", &valueLen);
                 if (clHeader) {
-                    contentLength = (uint32_t) strtoul(clHeader + 15, nullptr, 10);
+                    contentLength = (uint32_t) strtoul(clHeader, nullptr, 10);
                     if (contentLength > 0) {
-                        if (contentLength > 1024 * 1024) { // 1MB limit to prevent excessive memory allocation and potential attacks
+                        if (contentLength > FF_NETWORKING_MAX_RESPONSE_SIZE) { // 1MB limit to prevent excessive memory allocation and potential attacks
                             FF_DEBUG("Content-Length is too large: %u bytes, aborting", contentLength);
                             closesocket(state->sockfd);
                             state->sockfd = INVALID_SOCKET;
@@ -323,9 +357,47 @@ const char* ffNetworkingRecvHttpResponse(FFNetworkingState* state, FFstrbuf* buf
                         FF_DEBUG("Extended receive buffer to %u bytes", buffer->allocated);
                     }
                 }
+
+                // A chunked response has no Content-Length; it is framed by a last-chunk
+                const char* teHeader = ffNetworkingFindHeader(buffer->chars, headerEnd, "Transfer-Encoding:", &valueLen);
+                if (teHeader != nullptr) {
+                    switch (ffNetworkingParseTransferEncoding(teHeader, valueLen)) {
+                        case FF_NETWORKING_TE_CHUNKED:
+                            FF_DEBUG("Detected chunked transfer encoding");
+                            chunked = true;
+                            break;
+                        case FF_NETWORKING_TE_UNSUPPORTED:
+                            // The framing of e.g. `gzip, chunked` is unreadable and the payload
+                            // would stay encoded, so fail instead of returning garbage
+                            FF_DEBUG("Unsupported Transfer-Encoding: %.*s", (int) valueLen, teHeader);
+                            closesocket(state->sockfd);
+                            state->sockfd = INVALID_SOCKET;
+                            return "Unsupported Transfer-Encoding";
+                        default:
+                            break;
+                    }
+                }
             }
         }
-    } while (ffStrbufGetFree(buffer) > 0);
+
+        // Stop as soon as the response is framed, rather than waiting for the FIN
+        if (chunked) {
+            uint32_t consumed = 0;
+            int complete = ffNetworkingChunkedComplete(buffer->chars + headerEnd + 4, buffer->length - headerEnd - 4, &consumed);
+            if (complete < 0) {
+                FF_DEBUG("Malformed chunked body");
+                closesocket(state->sockfd);
+                state->sockfd = INVALID_SOCKET;
+                return "Malformed chunked body";
+            }
+            if (complete > 0) {
+                FF_DEBUG("Chunked body complete, %u bytes of encoded body", consumed);
+                break;
+            }
+        } else if (contentLength > 0 && buffer->length >= headerEnd + 4 + contentLength) {
+            break;
+        }
+    }
 
     FF_DEBUG("Closing socket: fd=%u", (unsigned) state->sockfd);
     closesocket(state->sockfd);
@@ -341,6 +413,10 @@ const char* ffNetworkingRecvHttpResponse(FFNetworkingState* state, FFstrbuf* buf
         return "No HTTP header end found";
     }
 
+    if (chunked && !ffNetworkingDecodeChunked(buffer, headerEnd)) {
+        return "Failed to decode chunked response";
+    }
+
     if (!ffStrbufStartsWithS(buffer, "HTTP/1.0 200 OK\r\n") && !ffStrbufStartsWithS(buffer, "HTTP/1.1 200 OK\r\n")) {
         FF_DEBUG("Invalid response: %.40s...", buffer->chars);
         return "Invalid response";

src/common/networking.h

@@ -29,9 +29,41 @@ typedef struct FFNetworkingState {
     bool tfo;         // if true, TCP Fast Open will be attempted first, and fallback to traditional connection if it fails
 } FFNetworkingState;
 
-const char* ffNetworkingSendHttpRequest(FFNetworkingState* state, const char* host, const char* path, const char* headers);
+const char* ffNetworkingSendHttpRequest(FFNetworkingState* state, const char* host, uint16_t port, const char* path, const char* headers);
 const char* ffNetworkingRecvHttpResponse(FFNetworkingState* state, FFstrbuf* buffer);
 
+// Case-insensitive header lookup restricted to the header block [0, headerEnd).
+// Restricting the range matters because the body may already share the same buffer.
+// Returns a pointer to the first character of the value; `valueLen` receives its
+// length excluding the terminating CRLF. Returns nullptr when the header is absent.
+const char* ffNetworkingFindHeader(const char* headers, uint32_t headerEnd, const char* name, uint32_t* valueLen);
+
+// Checks whether a `Transfer-Encoding: chunked` body has been received in full, so that
+// framing does not have to rely on the server closing the connection.
+// Returns 1 when complete (`consumed` receives the body length including trailers),
+// 0 when more data is needed, and -1 when the body is malformed.
+// The caller must not wait for a fixed amount of data (e.g. `MSG_WAITALL`) while the
+// response length is still unknown, otherwise this check never gets to run.
+int ffNetworkingChunkedComplete(const char* body, uint32_t bodyLen, uint32_t* consumed);
+
+// Decodes a `Transfer-Encoding: chunked` body in place and rewrites the response with a
+// `Content-Length` header in place of `Transfer-Encoding`.
+bool ffNetworkingDecodeChunked(FFstrbuf* buffer, uint32_t headerEnd);
+
+// Result of parsing a `Transfer-Encoding` header value
+typedef enum FFNetworkingTransferEncoding {
+    FF_NETWORKING_TE_NONE,        // the value holds no coding at all
+    FF_NETWORKING_TE_CHUNKED,     // exactly `chunked`, the only framing this client decodes
+    FF_NETWORKING_TE_UNSUPPORTED, // another coding or a chain of them, e.g. `gzip, chunked`
+} FFNetworkingTransferEncoding;
+
+// Parses a `Transfer-Encoding` header value. Codings are applied in the order they are
+// listed, so `chunked` has to be the last one for the framing to be readable at all --
+// and any other coding (e.g. `gzip, chunked`) leaves the payload encoded, which this
+// client cannot decode. Only a lone `chunked` is accepted; everything else is reported
+// as unsupported so that the caller fails the response instead of returning garbage.
+FFNetworkingTransferEncoding ffNetworkingParseTransferEncoding(const char* value, uint32_t valueLen);
+
 #ifdef FF_HAVE_ZLIB
 const char* ffNetworkingLoadZlibLibrary(void);
 bool ffNetworkingDecompressGzip(FFstrbuf* buffer, char* headerEnd);

src/detection/publicip/publicip.c

@@ -5,6 +5,20 @@
 static FFNetworkingState states[2];
 static const char* statuses[2] = { FF_UNINITIALIZED, FF_UNINITIALIZED };
 
+// Reads the port that follows the colon at `colonIndex`. The port has to be the last thing in
+// `host`, so this must run before any part of the host is trimmed off.
+static uint16_t ffPublicIpParseUrlPort(const FFstrbuf* host, uint32_t colonIndex) {
+    const char* portStr = host->chars + colonIndex + 1;
+    char* portEnd = nullptr;
+    unsigned long portValue = strtoul(portStr, &portEnd, 10);
+    if (portEnd == portStr || *portEnd != '\0' || portValue == 0 || portValue > 65535) {
+        fputs("Error: invalid port in the PublicIp module URL\n", stderr);
+        exit(1);
+    }
+
+    return (uint16_t) portValue;
+}
+
 void ffPreparePublicIp(FFPublicIPOptions* options) {
     FFNetworkingState* state = &states[options->ipv6];
     const char** status = &statuses[options->ipv6];
@@ -19,7 +33,7 @@ void ffPreparePublicIp(FFPublicIPOptions* options) {
     if (options->url.length == 0) {
         state->compression = true;
         state->tfo = true;
-        *status = ffNetworkingSendHttpRequest(state, options->ipv6 ? "v6.ipinfo.io" : "ipinfo.io", "/json", nullptr);
+        *status = ffNetworkingSendHttpRequest(state, options->ipv6 ? "v6.ipinfo.io" : "ipinfo.io", 80, "/json", nullptr);
     } else {
         FF_STRBUF_AUTO_DESTROY host = ffStrbufCreateCopy(&options->url);
         uint32_t hostStartIndex = ffStrbufFirstIndexS(&host, "://");
@@ -38,7 +52,39 @@ void ffPreparePublicIp(FFPublicIPOptions* options) {
             ffStrbufSubstrBefore(&host, pathStartIndex);
         }
 
-        *status = ffNetworkingSendHttpRequest(state, host.chars, path.length == 0 ? "/" : path.chars, nullptr);
+        // An optional `:port` must be split off the host, otherwise getaddrinfo() is asked to
+        // resolve a host name that still carries the port. Only a single colon can separate a
+        // port: a bare IPv6 literal (`::1`) holds several of them and has no port at all,
+        // which is why a bracketed literal (`[::1]:8080`) is the only unambiguous spelling.
+        uint16_t port = 0;
+        if (ffStrbufStartsWithC(&host, '[')) {
+            uint32_t bracketEnd = ffStrbufFirstIndexC(&host, ']');
+            if (bracketEnd == host.length) {
+                fputs("Error: unmatched '[' in the PublicIp module URL\n", stderr);
+                exit(1);
+            }
+
+            if (bracketEnd + 1 < host.length) {
+                if (host.chars[bracketEnd + 1] != ':') {
+                    fputs("Error: unexpected characters after the IPv6 literal in the PublicIp module URL\n", stderr);
+                    exit(1);
+                }
+                // Read the port while the string is still intact: trimming the brackets first
+                // would invalidate the index it was found at.
+                port = ffPublicIpParseUrlPort(&host, bracketEnd + 1);
+            }
+
+            ffStrbufSubstrBefore(&host, bracketEnd);
+            ffStrbufSubstrAfter(&host, 0); // drop the leading '['
+        } else {
+            uint32_t firstColon = ffStrbufFirstIndexC(&host, ':');
+            if (firstColon < host.length && firstColon == ffStrbufLastIndexC(&host, ':')) {
+                port = ffPublicIpParseUrlPort(&host, firstColon);
+                ffStrbufSubstrBefore(&host, firstColon);
+            }
+        }
+
+        *status = ffNetworkingSendHttpRequest(state, host.chars, port ?: 80, path.length == 0 ? "/" : path.chars, nullptr);
     }
 }
 

src/detection/weather/weather.c

@@ -29,7 +29,7 @@ void ffPrepareWeather(FFWeatherOptions* options) {
         default:
             break;
     }
-    status = ffNetworkingSendHttpRequest(&state, "wttr.in", path.chars, "User-Agent: curl/0.0.0\r\n");
+    status = ffNetworkingSendHttpRequest(&state, "wttr.in", 80, path.chars, "User-Agent: curl/0.0.0\r\n");
 }
 
 const char* ffDetectWeather(FFWeatherOptions* options, FFstrbuf* result) {