Skip to content

Commit 44ec925

Browse files
eqrioneqrion
committed
Decompress .z files in prefetch
1 parent ba180b5 commit 44ec925

File tree

3 files changed

+186
-9
lines changed

3 files changed

+186
-9
lines changed

‎JetStreamDriver.js‎

Lines changed: 132 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -81,8 +81,18 @@ if (typeof(URLSearchParams) !== "undefined") {
8181
globalThis.prefetchResources = getBoolParam(urlParameters, "prefetchResources");
8282
}
8383

84+
if (!isInBrowser && globalThis.prefetchResources) {
85+
// Use the wasm compiled zlib as a polyfill when decompression stream is
86+
// not available in JS shells.
87+
load("./wasm/zlib/shell.js");
88+
89+
// Load a polyfill for TextEncoder/TextDecoder in shells. Used when
90+
// decompressing a prefetched resource and converting it to text.
91+
load("./polyfills/fast-text-encoding/1.0.3/text.js");
92+
}
93+
8494
if (!globalThis.prefetchResources)
85-
console.warn("Disabling resource prefetching!");
95+
console.warn("Disabling resource prefetching! All compressed files must have been decompressed using `node utils/compress.mjs -d`");
8696

8797
// Used for the promise representing the current benchmark run.
8898
this.currentResolve = null;
@@ -189,6 +199,21 @@ function uiFriendlyDuration(time) {
189199
return `${time.toFixed(3)} ms`;
190200
}
191201

202+
// Files can be zlib compressed to reduce the size of the JetStream source code.
203+
// We don't use http compression because we support running from the shell and
204+
// don't want to require a complicated server setup.
205+
//
206+
// zlib was chosen because we already have it in tree for the wasm-zlib test.
207+
function isCompressed(name) {
208+
return name.endsWith(".z");
209+
}
210+
211+
function uncompressedName(name) {
212+
if (name.endsWith(".z"))
213+
return name.slice(0, -2);
214+
return name;
215+
}
216+
192217
// TODO: Cleanup / remove / merge. This is only used for caching loads in the
193218
// non-browser setting. In the browser we use exclusively `loadCache`,
194219
// `loadBlob`, `doLoadBlob`, `prefetchResourcesForBrowser` etc., see below.
@@ -201,14 +226,28 @@ class ShellFileLoader {
201226
// share common code.
202227
load(url) {
203228
console.assert(!isInBrowser);
229+
230+
let compressed = isCompressed(url);
231+
if (compressed && !globalThis.prefetchResources) {
232+
url = uncompressedName(url);
233+
}
234+
235+
// If we aren't supposed to prefetch this then return code snippet that will load the url on-demand.
204236
if (!globalThis.prefetchResources)
205237
return `load("${url}");`
206238

207239
if (this.requests.has(url)) {
208240
return this.requests.get(url);
209241
}
210242

211-
const contents = readFile(url);
243+
let contents;
244+
if (compressed) {
245+
let bytes = new Int8Array(read(url, "binary"));
246+
bytes = zlib.decompress(bytes);
247+
contents = new TextDecoder().decode(bytes);
248+
} else {
249+
contents = readFile(url);
250+
}
212251
this.requests.set(url, contents);
213252
return contents;
214253
}
@@ -260,10 +299,14 @@ class Driver {
260299
performance.mark("update-ui");
261300
benchmark.updateUIAfterRun();
262301

263-
if (isInBrowser && globalThis.prefetchResources) {
302+
if (isInBrowser) {
264303
const cache = JetStream.blobDataCache;
265304
for (const file of benchmark.files) {
266305
const blobData = cache[file];
306+
// If we didn't prefetch this resource, then no need to free it
307+
if (!blobData.blob) {
308+
continue
309+
}
267310
blobData.refCount--;
268311
if (!blobData.refCount)
269312
cache[file] = undefined;
@@ -412,6 +455,9 @@ class Driver {
412455

413456
async prefetchResources() {
414457
if (!isInBrowser) {
458+
if (globalThis.prefetchResources) {
459+
await zlib.initialize();
460+
}
415461
for (const benchmark of this.benchmarks)
416462
benchmark.prefetchResourcesForShell();
417463
return;
@@ -626,6 +672,11 @@ class Scripts {
626672
}
627673

628674
class ShellScripts extends Scripts {
675+
constructor() {
676+
super();
677+
this.prefetchedResources = [];
678+
}
679+
629680
run() {
630681
let globalObject;
631682
let realm;
@@ -652,13 +703,33 @@ class ShellScripts extends Scripts {
652703
currentReject
653704
};
654705

706+
// Pass the prefetched resources to the benchmark global.
707+
if (globalThis.prefetchResources) {
708+
// Pass the 'TextDecoder' polyfill into the benchmark global. Don't
709+
// use 'TextDecoder' as that will get picked up in the kotlin test
710+
// without full support.
711+
globalObject.ShellTextDecoder = TextDecoder;
712+
// Store shellPrefetchedResources on ShellPrefetchedResources so that
713+
// getBinary and getString can find them.
714+
globalObject.ShellPrefetchedResources = {};
715+
for (const [name, value] of this.prefetchedResources) {
716+
globalObject.ShellPrefetchedResources[name] = value;
717+
}
718+
} else {
719+
console.assert(this.prefetchedResources.length === 0, "Unexpected prefetched resources");
720+
}
721+
655722
globalObject.performance ??= performance;
656723
for (const script of this.scripts)
657724
globalObject.loadString(script);
658725

659726
return isD8 ? realm : globalObject;
660727
}
661728

729+
addPrefetchedResources(prefetchedResources) {
730+
this.prefetchedResources.push(...prefetchedResources);
731+
}
732+
662733
add(text) {
663734
this.scripts.push(text);
664735
}
@@ -691,7 +762,6 @@ class BrowserScripts extends Scripts {
691762
return magicFrame;
692763
}
693764

694-
695765
add(text) {
696766
this.scripts.push(`<script>${text}</script>`);
697767
}
@@ -711,6 +781,7 @@ class Benchmark {
711781
this.allowUtf16 = !!plan.allowUtf16;
712782
this.scripts = null;
713783
this.preloads = null;
784+
this.shellPrefetchedResources = null;
714785
this.results = [];
715786
this._state = BenchmarkState.READY;
716787
}
@@ -824,6 +895,9 @@ class Benchmark {
824895
if (!!this.plan.exposeBrowserTest)
825896
scripts.addBrowserTest();
826897

898+
if (this.shellPrefetchedResources) {
899+
scripts.addPrefetchedResources(this.shellPrefetchedResources);
900+
}
827901
if (this.plan.preload) {
828902
let preloadCode = "";
829903
for (let [ variableName, blobURLOrPath ] of this.preloads)
@@ -842,7 +916,7 @@ class Benchmark {
842916
} else {
843917
const cache = JetStream.blobDataCache;
844918
for (const file of this.plan.files) {
845-
scripts.addWithURL(globalThis.prefetchResources ? cache[file].blobURL : file);
919+
scripts.addWithURL(cache[file].blobURL);
846920
}
847921
}
848922

@@ -893,10 +967,19 @@ class Benchmark {
893967

894968
async doLoadBlob(resource) {
895969
const blobData = JetStream.blobDataCache[resource];
970+
971+
const compressed = isCompressed(resource);
972+
if (compressed && !globalThis.prefetchResources) {
973+
resource = uncompressedName(resource);
974+
}
975+
976+
// If we aren't supposed to prefetch this then set the blobURL to just
977+
// be the resource URL.
896978
if (!globalThis.prefetchResources) {
897979
blobData.blobURL = resource;
898980
return blobData;
899981
}
982+
900983
let response;
901984
let tries = 3;
902985
while (tries--) {
@@ -912,7 +995,15 @@ class Benchmark {
912995
continue;
913996
throw new Error("Fetch failed");
914997
}
915-
const blob = await response.blob();
998+
999+
// If we need to decompress this, then run it through a decompression
1000+
// stream.
1001+
if (compressed) {
1002+
const stream = response.body.pipeThrough(new DecompressionStream('deflate'))
1003+
response = new Response(stream);
1004+
}
1005+
1006+
let blob = await response.blob();
9161007
blobData.blob = blob;
9171008
blobData.blobURL = URL.createObjectURL(blob);
9181009
return blobData;
@@ -1048,7 +1139,28 @@ class Benchmark {
10481139
this.scripts = this.plan.files.map(file => shellFileLoader.load(file));
10491140

10501141
console.assert(this.preloads === null, "This initialization should be called only once.");
1051-
this.preloads = Object.entries(this.plan.preload ?? {});
1142+
this.preloads = [];
1143+
this.shellPrefetchedResources = [];
1144+
if (this.plan.preload) {
1145+
for (let name of Object.getOwnPropertyNames(this.plan.preload)) {
1146+
let file = this.plan.preload[name];
1147+
1148+
const compressed = isCompressed(file);
1149+
if (compressed && !globalThis.prefetchResources) {
1150+
file = uncompressedName(file);
1151+
}
1152+
1153+
if (globalThis.prefetchResources) {
1154+
let bytes = new Int8Array(read(file, "binary"));
1155+
if (compressed) {
1156+
bytes = zlib.decompress(bytes);
1157+
}
1158+
this.shellPrefetchedResources.push([file, bytes]);
1159+
}
1160+
1161+
this.preloads.push([name, file]);
1162+
}
1163+
}
10521164
}
10531165

10541166
scoreIdentifiers() {
@@ -1288,15 +1400,23 @@ class AsyncBenchmark extends DefaultBenchmark {
12881400
} else {
12891401
str += `
12901402
JetStream.getBinary = async function(path) {
1403+
if (ShellPrefetchedResources) {
1404+
return ShellPrefetchedResources[path];
1405+
}
12911406
return new Int8Array(read(path, "binary"));
12921407
};
12931408
12941409
JetStream.getString = async function(path) {
1410+
if (ShellPrefetchedResources) {
1411+
return new ShellTextDecoder().decode(ShellPrefetchedResources[path]);
1412+
}
12951413
return read(path);
12961414
};
12971415
12981416
JetStream.dynamicImport = async function(path) {
12991417
try {
1418+
// TODO: this skips the prefetched resources, but I'm
1419+
// not sure of a way around that.
13001420
return await import(path);
13011421
} catch (e) {
13021422
// In shells, relative imports require different paths, so try with and
@@ -1513,7 +1633,11 @@ class WasmLegacyBenchmark extends Benchmark {
15131633
`;
15141634
} else {
15151635
str += `
1516-
Module[key] = new Int8Array(read(path, "binary"));
1636+
if (ShellPrefetchedResources) {
1637+
Module[key] = ShellPrefetchedResources[path];
1638+
} else {
1639+
Module[key] = new Int8Array(read(path, "binary"));
1640+
}
15171641
if (andThen == doRun) {
15181642
globalObject.read = (...args) => {
15191643
console.log("should not be inside read: ", ...args);

‎wasm/zlib/shell.js‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
// zlib-based utility for use in shells where CompressionStream and
2+
// DecompressionStream are not available.
3+
4+
function module() {
5+
'use strict';
6+
7+
let zlibPromise = null;
8+
let zlibModule = null;
9+
10+
async function initialize() {
11+
if (zlibPromise) {
12+
zlibModule = await zlibPromise;
13+
return zlibModule;
14+
}
15+
load('wasm/zlib/build/zlib.js');
16+
zlibPromise = setupModule({
17+
wasmBinary: new Int8Array(read('wasm/zlib/build/zlib.wasm', "binary")),
18+
});
19+
zlibModule = await zlibPromise;
20+
return zlibModule;
21+
}
22+
23+
function decompress(bytes) {
24+
zlibModule.FS.writeFile('in', bytes);
25+
const inputzStr = zlibModule.stringToNewUTF8('in');
26+
const inputzoutStr = zlibModule.stringToNewUTF8('out');
27+
if (zlibModule._decompressFile(inputzStr, inputzoutStr) !== 0) {
28+
throw new Error();
29+
}
30+
const output = zlibModule.FS.readFile('out');
31+
zlibModule._free(inputzStr);
32+
zlibModule._free(inputzoutStr);
33+
return output;
34+
}
35+
36+
return {
37+
initialize: initialize,
38+
decompress: decompress,
39+
};
40+
}
41+
42+
globalThis.zlib = module();
43+

‎web-tooling-benchmark/benchmark.js‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,8 +38,18 @@ class Benchmark {
3838
async loadAllFiles(preload) {
3939
const loadPromises = Object.entries(preload).map(
4040
async ([name, url]) => {
41+
console.log(name);
42+
console.log(url);
4143
if (name.endsWith(".wasm")) {
42-
this.files[name] = (await JetStream.getBinary(url)).buffer;
44+
let buffer = (await JetStream.getBinary(url)).buffer;
45+
if (!(buffer instanceof ArrayBuffer)) {
46+
// The returned array buffer is from a different global when
47+
// prefetching resources and running in the shell. This is fine,
48+
// except for the source map code doing an instanceof
49+
// check that fails for the prototype being in a different realm.
50+
Object.setPrototypeOf(buffer, ArrayBuffer.prototype);
51+
}
52+
this.files[name] = buffer;
4353
} else {
4454
this.files[name] = await JetStream.getString(url);
4555
}

0 commit comments

Comments
 (0)