Skip to content

Commit 1d93124

Browse files
committed
[fetch] Support streaming fetch requests.
Add a new `FETCH_BACKEND` setting that can either be 'xhr' or 'fetch' to use the corresponding DOM API. To keep the emscripten Fetch.js code the same I've implemented a polyfill of XMLHttpRequest using Fetch. To support streaming I wired up the fetch code to use the old onprogress code that old versions of Firefox supported. Most of the current API is supported with some notable exceptions: - synchronous requests - overriding the mime type I also changed a few of the tests to support both sync and async so I could test them with the fetch backend.
1 parent 70b1876 commit 1d93124

File tree

7 files changed

+309
-13
lines changed

7 files changed

+309
-13
lines changed

src/Fetch.js

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -267,7 +267,11 @@ function fetchXHR(fetch, onsuccess, onerror, onprogress, onreadystatechange) {
267267
var userNameStr = userName ? UTF8ToString(userName) : undefined;
268268
var passwordStr = password ? UTF8ToString(password) : undefined;
269269

270+
#if FETCH_BACKEND == 'xhr'
270271
var xhr = new XMLHttpRequest();
272+
#else
273+
var xhr = new FetchXHR();
274+
#endif
271275
xhr.withCredentials = !!{{{ makeGetValue('fetch_attr', C_STRUCTS.emscripten_fetch_attr_t.withCredentials, 'u8') }}};;
272276
#if FETCH_DEBUG
273277
dbg(`fetch: xhr.timeout: ${xhr.timeout}, xhr.withCredentials: ${xhr.withCredentials}`);
@@ -276,8 +280,8 @@ function fetchXHR(fetch, onsuccess, onerror, onprogress, onreadystatechange) {
276280
xhr.open(requestMethod, url_, !fetchAttrSynchronous, userNameStr, passwordStr);
277281
if (!fetchAttrSynchronous) xhr.timeout = timeoutMsecs; // XHR timeout field is only accessible in async XHRs, and must be set after .open() but before .send().
278282
xhr.url_ = url_; // Save the url for debugging purposes (and for comparing to the responseURL that server side advertised)
279-
#if ASSERTIONS
280-
assert(!fetchAttrStreamData, 'streaming uses moz-chunked-arraybuffer which is no longer supported; TODO: rewrite using fetch()');
283+
#if ASSERTIONS && FETCH_BACKEND != 'fetch'
284+
assert(!fetchAttrStreamData, 'streaming is only supported when using the fetch backend');
281285
#endif
282286
xhr.responseType = 'arraybuffer';
283287

src/lib/libfetch.js

Lines changed: 239 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,245 @@ var LibraryFetch = {
3838
'$fetchLoadCachedData',
3939
'$fetchDeleteCachedData',
4040
#endif
41-
]
41+
#if FETCH_BACKEND == 'fetch'
42+
'$FetchXHR',
43+
#endif
44+
],
45+
/**
46+
* A class that mimics the XMLHttpRequest API using the modern Fetch API.
47+
* This implementation is specifically tailored to only handle 'arraybuffer'
48+
* responses.
49+
*/
50+
$FetchXHR: class {
51+
constructor() {
52+
// --- Public XHR Properties ---
53+
54+
// Event Handlers
55+
this.onload = null;
56+
this.onerror = null;
57+
this.onprogress = null;
58+
this.onreadystatechange = null;
59+
this.ontimeout = null;
60+
61+
// Request Configuration
62+
this.responseType = 'arraybuffer';
63+
this.withCredentials = false;
64+
this.timeout = 0; // Standard XHR timeout property
65+
66+
// Response / State Properties
67+
this.readyState = 0; // 0: UNSENT
68+
this.response = null;
69+
this.responseURL = '';
70+
this.status = 0;
71+
this.statusText = '';
72+
73+
// --- Internal Properties ---
74+
this._method = '';
75+
this._url = '';
76+
this._headers = {};
77+
this._abortController = null;
78+
this._aborted = false;
79+
this._responseHeaders = null;
80+
}
81+
82+
// --- Private state management ---
83+
_changeReadyState(state) {
84+
this.readyState = state;
85+
this.onreadystatechange?.();
86+
}
87+
88+
// --- Public XHR Methods ---
89+
90+
/**
91+
* Initializes a request.
92+
* @param {string} method The HTTP request method (e.g., 'GET', 'POST').
93+
* @param {string} url The URL to send the request to.
94+
* @param {boolean} [async=true] This parameter is ignored as Fetch is always async.
95+
* @param {string|null} [user=null] The username for basic authentication.
96+
* @param {string|null} [password=null] The password for basic authentication.
97+
*/
98+
open(method, url, async = true, user = null, password = null) {
99+
if (this.readyState !== 0 && this.readyState !== 4) {
100+
console.warn("FetchXHR.open() called while a request is in progress.");
101+
this.abort();
102+
}
103+
104+
// Reset internal state for the new request
105+
this._method = method;
106+
this._url = url;
107+
this._headers = {};
108+
this._responseHeaders = null;
109+
110+
// The async parameter is part of the XHR API but is an error here because
111+
// the Fetch API is inherently asynchronous and does not support synchronous requests.
112+
if (!async) {
113+
throw new Error("FetchXHR does not support synchronous requests.");
114+
}
115+
116+
// Handle Basic Authentication if user/password are provided.
117+
// This creates a base64-encoded string and sets the Authorization header.
118+
if (user) {
119+
const credentials = btoa(`${user}:${password || ''}`);
120+
this._headers['Authorization'] = `Basic ${credentials}`;
121+
}
122+
123+
this._changeReadyState(1); // 1: OPENED
124+
}
125+
126+
/**
127+
* Sets the value of an HTTP request header.
128+
* @param {string} header The name of the header.
129+
* @param {string} value The value of the header.
130+
*/
131+
setRequestHeader(header, value) {
132+
if (this.readyState !== 1) {
133+
throw new Error('setRequestHeader can only be called when state is OPENED.');
134+
}
135+
this._headers[header] = value;
136+
}
137+
138+
/**
139+
* This method is not effectively implemented because Fetch API relies on the
140+
* server's Content-Type header and does not support overriding the MIME type
141+
* on the client side in the same way as XHR.
142+
* @param {string} mimetype The MIME type to use.
143+
*/
144+
overrideMimeType(mimetype) {
145+
throw new Error("overrideMimeType is not supported by the Fetch API and has no effect.");
146+
}
147+
148+
/**
149+
* Returns a string containing all the response headers, separated by CRLF.
150+
* @returns {string} The response headers.
151+
*/
152+
getAllResponseHeaders() {
153+
if (!this._responseHeaders) {
154+
return '';
155+
}
156+
157+
let headersString = '';
158+
// The Headers object is iterable.
159+
for (const [key, value] of this._responseHeaders.entries()) {
160+
headersString += `${key}: ${value}\r\n`;
161+
}
162+
return headersString;
163+
}
164+
165+
/**
166+
* Sends the request.
167+
* @param {any} body The body of the request.
168+
*/
169+
async send(body = null) {
170+
if (this.readyState !== 1) {
171+
throw new Error('send() can only be called when state is OPENED.');
172+
}
173+
174+
this._abortController = new AbortController();
175+
const signal = this._abortController.signal;
176+
177+
// Handle timeout
178+
let timeoutID;
179+
if (this.timeout > 0) {
180+
timeoutID = setTimeout(
181+
() => this._abortController.abort(new DOMException('The user aborted a request.', 'TimeoutError')),
182+
this.timeout
183+
);
184+
}
185+
186+
const fetchOptions = {
187+
method: this._method,
188+
headers: this._headers,
189+
body: body,
190+
signal: signal,
191+
credentials: this.withCredentials ? 'include' : 'same-origin',
192+
};
193+
194+
try {
195+
const response = await fetch(this._url, fetchOptions);
196+
197+
// Populate response properties once headers are received
198+
this.status = response.status;
199+
this.statusText = response.statusText;
200+
this.responseURL = response.url;
201+
this._responseHeaders = response.headers;
202+
this._changeReadyState(2); // 2: HEADERS_RECEIVED
203+
204+
// Start processing the body
205+
this._changeReadyState(3); // 3: LOADING
206+
207+
if (!response.body) {
208+
throw new Error("Response has no body to read.");
209+
}
210+
211+
const reader = response.body.getReader();
212+
const contentLength = +response.headers.get('Content-Length');
213+
214+
let receivedLength = 0;
215+
const chunks = [];
216+
217+
while (true) {
218+
const { done, value } = await reader.read();
219+
if (done) {
220+
break;
221+
}
222+
223+
chunks.push(value);
224+
receivedLength += value.length;
225+
226+
if (this.onprogress) {
227+
// Convert to ArrayBuffer as requested by responseType.
228+
this.response = value.buffer;
229+
const progressEvent = {
230+
lengthComputable: contentLength > 0,
231+
loaded: receivedLength,
232+
total: contentLength
233+
};
234+
this.onprogress(progressEvent);
235+
}
236+
}
237+
238+
// Combine chunks into a single Uint8Array.
239+
const allChunks = new Uint8Array(receivedLength);
240+
let position = 0;
241+
for (const chunk of chunks) {
242+
allChunks.set(chunk, position);
243+
position += chunk.length;
244+
}
245+
246+
// Convert to ArrayBuffer as requested by responseType
247+
this.response = allChunks.buffer;
248+
} catch (error) {
249+
this.statusText = error.message;
250+
251+
if (error.name === 'AbortError') {
252+
// Do nothing.
253+
} else if (error.name === 'TimeoutError') {
254+
this.ontimeout?.();
255+
} else {
256+
// This is a network error
257+
this.onerror?.();
258+
}
259+
} finally {
260+
clearTimeout(timeoutID);
261+
if (!this._aborted) {
262+
this._changeReadyState(4); // 4: DONE
263+
// The XHR 'load' event fires for successful HTTP statuses (2xx) as well as
264+
// unsuccessful ones (4xx, 5xx). The 'error' event is for network failures.
265+
this.onload?.();
266+
}
267+
}
268+
}
269+
270+
/**
271+
* Aborts the request if it has already been sent.
272+
*/
273+
abort() {
274+
this._aborted = true;
275+
this.status = 0;
276+
this._changeReadyState(4); // 4: DONE
277+
this._abortController?.abort();
278+
}
279+
}
42280
};
43281

44282
addToLibrary(LibraryFetch);

src/settings.js

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1834,6 +1834,17 @@ var FETCH_DEBUG = false;
18341834
// [link]
18351835
var FETCH = false;
18361836

1837+
// Change which DOM API is used for the emscripten fetch API. Valid options are:
1838+
// - 'xhr' (default) uses XMLHttpRequest
1839+
// - 'fetch' uses Fetch
1840+
// Both options generally support the same API, but there are some key
1841+
// differences:
1842+
// - XHR supports synchronous requests
1843+
// - XHR supports overriding mime types
1844+
// - Fetch supports streaming data using the 'onprogress' callback
1845+
// [link]
1846+
var FETCH_BACKEND = 'xhr';
1847+
18371848
// ATTENTION [WIP]: Experimental feature. Please use at your own risk.
18381849
// This will eventually replace the current JS file system implementation.
18391850
// If set to 1, uses new filesystem implementation.

test/common.py

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -697,6 +697,20 @@ def metafunc(self, with_wasm2js, *args, **kwargs):
697697
return metafunc
698698

699699

700+
def also_with_fetch_backend(f):
701+
assert callable(f)
702+
703+
@wraps(f)
704+
def metafunc(self, with_fetch, *args, **kwargs):
705+
if with_fetch:
706+
self.set_setting('FETCH_BACKEND', 'fetch')
707+
f(self, *args, **kwargs)
708+
709+
parameterize(metafunc, {'': (False,),
710+
'fetch_backend': (True,)})
711+
return metafunc
712+
713+
700714
def can_do_standalone(self, impure=False):
701715
# Pure standalone engines don't support MEMORY64 yet. Even with MEMORY64=2 (lowered)
702716
# the WASI APIs that take pointer values don't have 64-bit variants yet.

test/fetch/test_fetch_redirect.c

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -69,11 +69,13 @@ void start_next_async_fetch() {
6969
async_method_idx++;
7070
if (async_method_idx >= num_methods) {
7171
// All async tests done, now run sync tests
72+
#ifndef SKIP_SYNC_TESTS
7273
for (int m = 0; m < num_methods; ++m) {
7374
for (int i = 0; i < num_codes; ++i) {
7475
fetchSyncTest(redirect_codes[i], methods[m]);
7576
}
7677
}
78+
#endif
7779
exit(0);
7880
}
7981
}

test/fetch/test_fetch_response_headers.cpp

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,10 @@ int main() {
2121
emscripten_fetch_attr_t attr;
2222
emscripten_fetch_attr_init(&attr);
2323
strcpy(attr.requestMethod, "GET");
24-
attr.attributes = EMSCRIPTEN_FETCH_REPLACE | EMSCRIPTEN_FETCH_LOAD_TO_MEMORY | EMSCRIPTEN_FETCH_SYNCHRONOUS;
24+
attr.attributes = EMSCRIPTEN_FETCH_REPLACE | EMSCRIPTEN_FETCH_LOAD_TO_MEMORY;
25+
#ifdef SYNC
26+
attr.attributes |= EMSCRIPTEN_FETCH_SYNCHRONOUS;
27+
#endif
2528
attr.requestHeaders = headers;
2629

2730
attr.onsuccess = [] (emscripten_fetch_t *fetch) {
@@ -44,6 +47,9 @@ int main() {
4447
printf("Data checksum: %02X\n", checksum);
4548
assert(checksum == 0x08);
4649
emscripten_fetch_close(fetch);
50+
#ifndef SYNC
51+
exit(0);
52+
#endif
4753

4854
if (result == 1) result = 0;
4955
};
@@ -63,9 +69,11 @@ int main() {
6369
};
6470

6571
emscripten_fetch_t *fetch = emscripten_fetch(&attr, "gears.png");
72+
#ifdef SYNC
6673
if (result != 0) {
6774
result = 2;
6875
printf("emscripten_fetch() failed to run synchronously!\n");
6976
}
77+
#endif
7078
return result;
7179
}

0 commit comments

Comments
 (0)