Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions AUTHORS
Original file line number Diff line number Diff line change
Expand Up @@ -601,3 +601,4 @@ a license to everyone to use it as detailed in LICENSE.)
* Artur Gatin <agatin@teladochealth.com> (copyright owned by Teladoc Health, Inc.)
* Christian Lloyd <clloyd@teladochealth.com> (copyright owned by Teladoc Health, Inc.)
* Sean Morris <sean@seanmorr.is>
* Pt. Prashant Tripathi <ptprashanttripathi@outlook.com>
12 changes: 12 additions & 0 deletions site/source/docs/tools_reference/settings_reference.rst
Original file line number Diff line number Diff line change
Expand Up @@ -2700,6 +2700,18 @@ of ENVIRONMENT since TextDecoder is not available in those environments).

Default value: 1

.. _textencoder:

TEXTENCODER
===========

The default value of 1 means the generated code will use TextEncoder if
available and fall back to custom encoding code when it is not available.
If set to 2, we assume TextEncoder is always present and usable, and no
fallback JS code will be emitted.

Default value: 1

.. _embind_std_string_is_utf8:

EMBIND_STD_STRING_IS_UTF8
Expand Down
51 changes: 50 additions & 1 deletion src/lib/libstrings.js
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,10 @@
#error "TEXTDECODER must be either 1 or 2"
#endif

#if TEXTENCODER != 1 && TEXTENCODER != 2
#error "TEXTENCODER must be either 1 or 2"
#endif

addToLibrary({
// TextDecoder constructor defaults to UTF-8
#if TEXTDECODER == 2
Expand All @@ -16,6 +20,13 @@ addToLibrary({
$UTF8Decoder: "typeof TextDecoder != 'undefined' ? new TextDecoder() : undefined",
#endif

// TextEncoder constructor defaults to UTF-8
#if TEXTENCODER == 2
$UTF8Encoder: "new TextEncoder()",
#else
$UTF8Encoder: "typeof TextEncoder != 'undefined' ? new TextEncoder() : undefined",
#endif

$findStringEnd: (heapOrArray, idx, maxBytesToRead, ignoreNul) => {
var maxIdx = idx + maxBytesToRead;
if (ignoreNul) return maxIdx;
Expand Down Expand Up @@ -147,9 +158,12 @@ addToLibrary({
* terminator.
* @return {number} The number of bytes written, EXCLUDING the null terminator.
*/
$stringToUTF8Array__deps: [
'$UTF8Encoder',
#if ASSERTIONS
$stringToUTF8Array__deps: ['$warnOnce'],
'$warnOnce',
#endif
],
$stringToUTF8Array: (str, heap, outIdx, maxBytesToWrite) => {
#if CAN_ADDRESS_2GB
outIdx >>>= 0;
Expand All @@ -162,6 +176,28 @@ addToLibrary({
if (!(maxBytesToWrite > 0))
return 0;

#if TEXTENCODER == 2
// Always use TextEncoder when TEXTENCODER == 2
var encoded = UTF8Encoder.encode(str);
var bytesToWrite = Math.min(encoded.length, maxBytesToWrite - 1); // -1 for null terminator
for (var i = 0; i < bytesToWrite; ++i) {
heap[outIdx + i] = encoded[i];
}
heap[outIdx + bytesToWrite] = 0;
return bytesToWrite;
#else
// When using conditional TextEncoder
if (UTF8Encoder) {
var encoded = UTF8Encoder.encode(str);
var bytesToWrite = Math.min(encoded.length, maxBytesToWrite - 1); // -1 for null terminator
for (var i = 0; i < bytesToWrite; ++i) {
heap[outIdx + i] = encoded[i];
}
heap[outIdx + bytesToWrite] = 0;
return bytesToWrite;
}

// Fallback: manual UTF-8 encoding
var startIdx = outIdx;
var endIdx = outIdx + maxBytesToWrite - 1; // -1 for string null terminator.
for (var i = 0; i < str.length; ++i) {
Expand Down Expand Up @@ -198,6 +234,7 @@ addToLibrary({
// Null-terminate the pointer to the buffer.
heap[outIdx] = 0;
return outIdx - startIdx;
#endif // TEXTENCODER == 2
},

/**
Expand All @@ -224,7 +261,18 @@ addToLibrary({
* @param {string} str - JavaScript string to operator on
* @return {number} Length, in bytes, of the UTF8 encoded string.
*/
$lengthBytesUTF8__deps: ['$UTF8Encoder'],
$lengthBytesUTF8: (str) => {
#if TEXTENCODER == 2
// Always use TextEncoder when TEXTENCODER == 2
return UTF8Encoder.encode(str).length;
#else
// When using conditional TextEncoder
if (UTF8Encoder) {
return UTF8Encoder.encode(str).length;
}

// Fallback: manual calculation
var len = 0;
for (var i = 0; i < str.length; ++i) {
// Gotcha: charCodeAt returns a 16-bit word that is a UTF-16 encoded code
Expand All @@ -243,6 +291,7 @@ addToLibrary({
}
}
return len;
#endif // TEXTENCODER == 2
},

$intArrayFromString__docs: '/** @type {function(string, boolean=, number=)} */',
Expand Down
7 changes: 7 additions & 0 deletions src/settings.js
Original file line number Diff line number Diff line change
Expand Up @@ -1776,6 +1776,13 @@ var EVAL_CTORS = 0;
// [link]
var TEXTDECODER = 1;

// The default value of 1 means the generated code will use TextEncoder if
// available and fall back to custom encoding code when it is not available.
// If set to 2, we assume TextEncoder is always present and usable, and no
// fallback JS code will be emitted.
// [link]
var TEXTENCODER = 1;

// Embind specific: If enabled, assume UTF-8 encoded data in std::string binding.
// Disable this to support binary data transfer.
// [link]
Expand Down
12 changes: 6 additions & 6 deletions test/code_size/test_codesize_cxx_ctors1.json
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
{
"a.out.js": 19754,
"a.out.js.gz": 8162,
"a.out.nodebug.wasm": 129509,
"a.out.nodebug.wasm.gz": 49243,
"total": 149263,
"total_gz": 57405,
"a.out.js": 19941,
"a.out.js.gz": 8224,
"a.out.nodebug.wasm": 129513,
"a.out.nodebug.wasm.gz": 49252,
"total": 149454,
"total_gz": 57476,
"sent": [
"__cxa_throw",
"_abort_js",
Expand Down
12 changes: 6 additions & 6 deletions test/code_size/test_codesize_cxx_ctors2.json
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
{
"a.out.js": 19732,
"a.out.js.gz": 8148,
"a.out.nodebug.wasm": 128936,
"a.out.nodebug.wasm.gz": 48884,
"total": 148668,
"total_gz": 57032,
"a.out.js": 19919,
"a.out.js.gz": 8209,
"a.out.nodebug.wasm": 128940,
"a.out.nodebug.wasm.gz": 48891,
"total": 148859,
"total_gz": 57100,
"sent": [
"__cxa_throw",
"_abort_js",
Expand Down
12 changes: 6 additions & 6 deletions test/code_size/test_codesize_cxx_except.json
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
{
"a.out.js": 23415,
"a.out.js.gz": 9145,
"a.out.nodebug.wasm": 171271,
"a.out.nodebug.wasm.gz": 57338,
"total": 194686,
"total_gz": 66483,
"a.out.js": 23604,
"a.out.js.gz": 9203,
"a.out.nodebug.wasm": 171275,
"a.out.nodebug.wasm.gz": 57346,
"total": 194879,
"total_gz": 66549,
"sent": [
"__cxa_begin_catch",
"__cxa_end_catch",
Expand Down
12 changes: 6 additions & 6 deletions test/code_size/test_codesize_cxx_except_wasm.json
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
{
"a.out.js": 19643,
"a.out.js.gz": 8112,
"a.out.nodebug.wasm": 144630,
"a.out.nodebug.wasm.gz": 54894,
"total": 164273,
"total_gz": 63006,
"a.out.js": 19831,
"a.out.js.gz": 8171,
"a.out.nodebug.wasm": 144634,
"a.out.nodebug.wasm.gz": 54899,
"total": 164465,
"total_gz": 63070,
"sent": [
"_abort_js",
"_tzset_js",
Expand Down
12 changes: 6 additions & 6 deletions test/code_size/test_codesize_cxx_except_wasm_legacy.json
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
{
"a.out.js": 19643,
"a.out.js.gz": 8112,
"a.out.nodebug.wasm": 142219,
"a.out.nodebug.wasm.gz": 54358,
"total": 161862,
"total_gz": 62470,
"a.out.js": 19831,
"a.out.js.gz": 8171,
"a.out.nodebug.wasm": 142223,
"a.out.nodebug.wasm.gz": 54362,
"total": 162054,
"total_gz": 62533,
"sent": [
"_abort_js",
"_tzset_js",
Expand Down
12 changes: 6 additions & 6 deletions test/code_size/test_codesize_cxx_lto.json
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
{
"a.out.js": 19082,
"a.out.js.gz": 7841,
"a.out.nodebug.wasm": 106463,
"a.out.nodebug.wasm.gz": 42596,
"total": 125545,
"total_gz": 50437,
"a.out.js": 18725,
"a.out.js.gz": 7678,
"a.out.nodebug.wasm": 106473,
"a.out.nodebug.wasm.gz": 42629,
"total": 125198,
"total_gz": 50307,
"sent": [
"a (emscripten_resize_heap)",
"b (_setitimer_js)",
Expand Down
12 changes: 6 additions & 6 deletions test/code_size/test_codesize_cxx_mangle.json
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
{
"a.out.js": 23465,
"a.out.js.gz": 9164,
"a.out.nodebug.wasm": 235312,
"a.out.nodebug.wasm.gz": 78933,
"total": 258777,
"total_gz": 88097,
"a.out.js": 23654,
"a.out.js.gz": 9223,
"a.out.nodebug.wasm": 235316,
"a.out.nodebug.wasm.gz": 78938,
"total": 258970,
"total_gz": 88161,
"sent": [
"__cxa_begin_catch",
"__cxa_end_catch",
Expand Down
12 changes: 6 additions & 6 deletions test/code_size/test_codesize_cxx_noexcept.json
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
{
"a.out.js": 19754,
"a.out.js.gz": 8162,
"a.out.nodebug.wasm": 131926,
"a.out.nodebug.wasm.gz": 50238,
"total": 151680,
"total_gz": 58400,
"a.out.js": 19941,
"a.out.js.gz": 8224,
"a.out.nodebug.wasm": 131930,
"a.out.nodebug.wasm.gz": 50245,
"total": 151871,
"total_gz": 58469,
"sent": [
"__cxa_throw",
"_abort_js",
Expand Down
12 changes: 6 additions & 6 deletions test/code_size/test_codesize_cxx_wasmfs.json
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
{
"a.out.js": 7143,
"a.out.js.gz": 3338,
"a.out.nodebug.wasm": 169797,
"a.out.nodebug.wasm.gz": 63087,
"total": 176940,
"total_gz": 66425,
"a.out.js": 7353,
"a.out.js.gz": 3407,
"a.out.nodebug.wasm": 169801,
"a.out.nodebug.wasm.gz": 63095,
"total": 177154,
"total_gz": 66502,
"sent": [
"__cxa_throw",
"_abort_js",
Expand Down
8 changes: 4 additions & 4 deletions test/code_size/test_codesize_file_preload.json
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
{
"a.out.js": 22684,
"a.out.js.gz": 9375,
"a.out.js": 22869,
"a.out.js.gz": 9431,
"a.out.nodebug.wasm": 1681,
"a.out.nodebug.wasm.gz": 960,
"total": 24365,
"total_gz": 10335,
"total": 24550,
"total_gz": 10391,
"sent": [
"a (fd_write)"
],
Expand Down
8 changes: 4 additions & 4 deletions test/code_size/test_codesize_files_js_fs.json
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
{
"a.out.js": 18357,
"a.out.js.gz": 7470,
"a.out.js": 18536,
"a.out.js.gz": 7528,
"a.out.nodebug.wasm": 381,
"a.out.nodebug.wasm.gz": 260,
"total": 18738,
"total_gz": 7730,
"total": 18917,
"total_gz": 7788,
"sent": [
"a (fd_write)",
"b (fd_read)",
Expand Down
10 changes: 5 additions & 5 deletions test/code_size/test_codesize_files_wasmfs.json
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
{
"a.out.js": 5549,
"a.out.js.gz": 2591,
"a.out.nodebug.wasm": 50231,
"a.out.js": 5759,
"a.out.js.gz": 2665,
"a.out.nodebug.wasm": 50237,
"a.out.nodebug.wasm.gz": 18094,
"total": 55780,
"total_gz": 20685,
"total": 55996,
"total_gz": 20759,
"sent": [
"a (emscripten_date_now)",
"b (emscripten_err)",
Expand Down
8 changes: 4 additions & 4 deletions test/code_size/test_codesize_hello_O0.json
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
{
"a.out.js": 22496,
"a.out.js.gz": 8322,
"a.out.js": 22508,
"a.out.js.gz": 8324,
"a.out.nodebug.wasm": 15127,
"a.out.nodebug.wasm.gz": 7450,
"total": 37623,
"total_gz": 15772,
"total": 37635,
"total_gz": 15774,
"sent": [
"fd_write"
],
Expand Down
8 changes: 4 additions & 4 deletions test/code_size/test_codesize_hello_dylink.json
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
{
"a.out.js": 26976,
"a.out.js.gz": 11458,
"a.out.js": 27164,
"a.out.js.gz": 11514,
"a.out.nodebug.wasm": 18567,
"a.out.nodebug.wasm.gz": 9199,
"total": 45543,
"total_gz": 20657,
"total": 45731,
"total_gz": 20713,
"sent": [
"__heap_base",
"__indirect_function_table",
Expand Down
6 changes: 3 additions & 3 deletions test/code_size/test_codesize_hello_dylink_all.json
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
{
"a.out.js": 246331,
"a.out.nodebug.wasm": 597763,
"total": 844094,
"a.out.js": 246519,
"a.out.nodebug.wasm": 597778,
"total": 844297,
"sent": [
"IMG_Init",
"IMG_Load",
Expand Down
12 changes: 6 additions & 6 deletions test/code_size/test_codesize_mem_O3.json
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
{
"a.out.js": 4399,
"a.out.js.gz": 2112,
"a.out.nodebug.wasm": 5260,
"a.out.nodebug.wasm.gz": 2418,
"total": 9659,
"total_gz": 4530,
"a.out.js": 4581,
"a.out.js.gz": 2194,
"a.out.nodebug.wasm": 5267,
"a.out.nodebug.wasm.gz": 2422,
"total": 9848,
"total_gz": 4616,
"sent": [
"a (emscripten_resize_heap)"
],
Expand Down
Loading