feat: add http/https agent support for connection pooling (#619)

Author: paveldudka

README.md

@@ -110,6 +110,47 @@ server.on('requestFailed', ({ request, error }) => {
 });
 ```
 
+## Use custom HTTP agents for connection pooling
+
+You can provide custom HTTP/HTTPS agents to enable connection pooling and reuse with upstream proxies. This is particularly useful for maintaining sticky IP addresses or reducing connection overhead:
+
+```javascript
+const http = require('http');
+const https = require('https');
+const ProxyChain = require('proxy-chain');
+
+// Create agents with keepAlive to enable connection pooling
+const httpAgent = new http.Agent({
+    keepAlive: true,
+    maxSockets: 10,
+});
+
+const httpsAgent = new https.Agent({
+    keepAlive: true,
+    maxSockets: 10,
+});
+
+const server = new ProxyChain.Server({
+    port: 8000,
+    prepareRequestFunction: ({ request }) => {
+        return {
+            upstreamProxyUrl: 'http://proxy.example.com:8080',
+            // Or for HTTPS upstream proxy: 'https://proxy.example.com:8080'
+
+            // Agents enable connection pooling to upstream proxy
+            httpAgent,    // Used for HTTP upstream proxies
+            httpsAgent,   // Used for HTTPS upstream proxies
+        };
+    },
+});
+
+server.listen(() => {
+    console.log(`Proxy server is listening on port ${server.port}`);
+});
+```
+
+**Note:** Custom agents are only supported for HTTP and HTTPS upstream proxies. SOCKS upstream proxies use direct socket connections and do not support custom agents.
+
 ## SOCKS support
 SOCKS protocol is supported for versions 4 and 5, specifically: `['socks', 'socks4', 'socks4a', 'socks5', 'socks5h']`, where `socks` will default to version 5.
 

package.json

@@ -1,6 +1,6 @@
 {
     "name": "proxy-chain",
-    "version": "2.5.9",
+    "version": "2.6.0",
     "description": "Node.js implementation of a proxy server (think Squid) with support for SSL, authentication, upstream proxy chaining, and protocol tunneling.",
     "main": "dist/index.js",
     "keywords": [

src/chain.ts

@@ -27,6 +27,8 @@ export interface HandlerOpts {
     ipFamily?: number;
     dnsLookup?: typeof dns['lookup'];
     customTag?: unknown;
+    httpAgent?: http.Agent;
+    httpsAgent?: https.Agent;
 }
 
 interface ChainOpts {
@@ -87,8 +89,12 @@ export const chain = (
         ? https.request(proxy.origin, {
             ...options,
             rejectUnauthorized: !handlerOpts.ignoreUpstreamProxyCertificate,
+            agent: handlerOpts.httpsAgent,
         })
-        : http.request(proxy.origin, options);
+        : http.request(proxy.origin, {
+            ...options,
+            agent: handlerOpts.httpAgent,
+        });
 
     client.once('socket', (targetSocket: SocketWithPreviousStats) => {
         // Socket can be re-used by multiple requests.

src/chain_socks.ts

@@ -34,6 +34,12 @@ const socksProtocolToVersionNumber = (protocol: string): 4 | 5 => {
 };
 
 /**
+ * Tunnels CONNECT requests through a SOCKS upstream proxy.
+ *
+ * **Note:** Custom HTTP/HTTPS agents (`httpAgent`, `httpsAgent`) from `prepareRequestFunction`
+ * are not supported with SOCKS upstream proxies. SOCKS establishes direct TCP socket connections
+ * using the SocksClient, which bypasses HTTP agent connection pooling.
+ *
  * Client -> Apify (CONNECT) -> Upstream (SOCKS) -> Web
  * Client <- Apify (CONNECT) <- Upstream (SOCKS) <- Web
  */

src/forward.ts

@@ -29,6 +29,8 @@ export interface HandlerOpts {
     localAddress?: string;
     ipFamily?: number;
     dnsLookup?: typeof dns['lookup'];
+    httpAgent?: http.Agent;
+    httpsAgent?: https.Agent;
 }
 
 /**
@@ -118,9 +120,13 @@ export const forward = async (
         ? https.request(origin!, {
             ...options as unknown as https.RequestOptions,
             rejectUnauthorized: handlerOpts.upstreamProxyUrlParsed ? !handlerOpts.ignoreUpstreamProxyCertificate : undefined,
+            agent: handlerOpts.httpsAgent,
         }, requestCallback)
 
-        : http.request(origin!, options as unknown as http.RequestOptions, requestCallback);
+        : http.request(origin!, {
+            ...options as unknown as http.RequestOptions,
+            agent: handlerOpts.httpAgent,
+        }, requestCallback);
 
     client.once('socket', (socket: SocketWithPreviousStats) => {
         // Socket can be re-used by multiple requests.

src/forward_socks.ts

@@ -26,6 +26,12 @@ export interface HandlerOpts {
 }
 
 /**
+ * Forwards HTTP requests through a SOCKS upstream proxy.
+ *
+ * **Note:** Custom HTTP/HTTPS agents (`httpAgent`, `httpsAgent`) from `prepareRequestFunction`
+ * are not supported with SOCKS upstream proxies. SOCKS uses direct socket connections
+ * managed by the SocksProxyAgent, which does not utilize HTTP agents.
+ *
  * ```
  * Client -> Apify (HTTP) -> Upstream (SOCKS) -> Web
  * Client <- Apify (HTTP) <- Upstream (SOCKS) <- Web

src/server.ts

@@ -3,6 +3,7 @@ import { Buffer } from 'node:buffer';
 import type dns from 'node:dns';
 import { EventEmitter } from 'node:events';
 import http from 'node:http';
+import type https from 'node:https';
 import type net from 'node:net';
 import { URL } from 'node:url';
 import util from 'node:util';
@@ -63,6 +64,8 @@ type HandlerOpts = {
     ipFamily?: number;
     dnsLookup?: typeof dns['lookup'];
     customTag?: unknown;
+    httpAgent?: http.Agent;
+    httpsAgent?: https.Agent;
 };
 
 export type PrepareRequestFunctionOpts = {
@@ -86,6 +89,8 @@ export type PrepareRequestFunctionResult = {
     ipFamily?: number;
     dnsLookup?: typeof dns['lookup'];
     customTag?: unknown;
+    httpAgent?: http.Agent;
+    httpsAgent?: https.Agent;
 };
 
 type Promisable<T> = T | Promise<T>;
@@ -453,6 +458,8 @@ export class Server extends EventEmitter {
         handlerOpts.dnsLookup = funcResult.dnsLookup;
         handlerOpts.customConnectServer = funcResult.customConnectServer;
         handlerOpts.customTag = funcResult.customTag;
+        handlerOpts.httpAgent = funcResult.httpAgent;
+        handlerOpts.httpsAgent = funcResult.httpsAgent;
 
         // If not authenticated, request client to authenticate
         if (funcResult.requestAuthentication) {

test/Dockerfile

@@ -1,6 +1,6 @@
 FROM node:18.20.8-bookworm@sha256:c6ae79e38498325db67193d391e6ec1d224d96c693a8a4d943498556716d3783
 
-RUN apt-get update && apt-get install -y --no-install-recommends chromium=140.0.7339.185-1~deb12u1 \
+RUN apt-get update && apt-get install -y --no-install-recommends chromium=142.0.7444.134-1~deb12u1 \
     && rm -rf /var/lib/apt/lists/*
 
 ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true