Skip to content

Update Lite3 to get C11 compiler fix, and decoding bug fix #15

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
mignon-p wants to merge 2 commits into
base: main
Choose a base branch
from
Open

Update Lite3 to get C11 compiler fix, and decoding bug fix #15

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
ef0dfe2
Update Lite3 library to 7b62398a35e9647f0d6bb709d714bd9cd2100182
mignon-p Mar 28, 2026
f2c022a
Comment out checks for C23 compiler
mignon-p Mar 28, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 6 additions & 4 deletions build.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,16 +5,16 @@
//! Invariants: Requests C23-compatible mode for vendored Lite3 sources that declare variables after labels.
//! Invariants: Uses only Cargo-provided env vars (e.g. `CARGO_MANIFEST_DIR`).
use std::env;
use std::fs;
use std::path::{Path, PathBuf};
// use std::fs;
use std::path::{/*Path,*/ PathBuf};

fn main() {
let target = env::var("TARGET").unwrap_or_default();
let manifest_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR").expect("CARGO_MANIFEST_DIR"));
let lite3_dir = manifest_dir.join("vendor").join("lite3");
let include_dir = lite3_dir.join("include");
let lib_dir = lite3_dir.join("lib");
let out_dir = PathBuf::from(env::var("OUT_DIR").expect("OUT_DIR"));
// let out_dir = PathBuf::from(env::var("OUT_DIR").expect("OUT_DIR"));

println!("cargo:rerun-if-changed=c/lite3_shim.c");
println!("cargo:rerun-if-changed=c/lite3_shim.h");
Expand All @@ -29,7 +29,7 @@ fn main() {
println!("cargo:rerun-if-changed=vendor/lite3/lib/nibble_base64/base64.c");
println!("cargo:rerun-if-changed=ui/index.html");

ensure_c23_label_decl_support(&target, &out_dir);
// ensure_c23_label_decl_support(&target, &out_dir);

let mut build = cc::Build::new();
build
Expand All @@ -49,6 +49,7 @@ fn main() {
build.compile("lite3");
}

/*
fn ensure_c23_label_decl_support(target: &str, out_dir: &Path) {
let probe_source = out_dir.join("lite3_c23_probe.c");
fs::write(
Expand Down Expand Up @@ -81,6 +82,7 @@ int lite3_c23_probe(int x) {
);
}
}
*/

fn configure_lite3_compiler(build: &mut cc::Build, target: &str) {
if target.contains("windows-msvc") {
Expand Down
4 changes: 2 additions & 2 deletions vendor/README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,8 @@ directly, rather than fetching at build time.
## Lite3

- Upstream: https://github.com/fastserial/lite3
- Pinned commit: ac7fc194612fb5d78a978e2c618be4d69fe0fcbb
- Pin date: 2026-01-30
- Pinned commit: 7b62398a35e9647f0d6bb709d714bd9cd2100182
- Pin date: 2026-03-28

### Update procedure

Expand Down
17 changes: 12 additions & 5 deletions vendor/lite3/include/lite3.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -326,7 +326,7 @@ Changing this setting also requires changing other settings. See `struct node` i

#ifndef DOXYGEN_IGNORE
#define LITE3_NODE_SIZE_SHIFT 6
#define LITE3_NODE_SIZE_MASK ((u32)~((1 << 6) - 1)) // 26 MSB
#define LITE3_NODE_SIZE_MASK ((uint32_t)~((1 << 6) - 1)) // 26 MSB

#define LITE3_DJB2_HASH_SEED ((uint32_t)5381)
#endif // DOXYGEN_IGNORE
Expand Down Expand Up @@ -1155,7 +1155,9 @@ static inline int _lite3_set_by_index(unsigned char *buf, size_t *__restrict ino
int ret;
if ((ret = _lite3_verify_arr_set(buf, inout_buflen, ofs, bufsz)) < 0)
return ret;
uint32_t size = (*(uint32_t *)(buf + ofs + LITE3_NODE_SIZE_KC_OFFSET)) >> LITE3_NODE_SIZE_SHIFT;
uint32_t size;
memcpy(&size, buf + ofs + LITE3_NODE_SIZE_KC_OFFSET, sizeof(size));
size >>= LITE3_NODE_SIZE_SHIFT;
if (LITE3_UNLIKELY(index > size)) {
LITE3_PRINT_ERROR("INVALID ARGUMENT: ARRAY INDEX %u OUT OF BOUNDS (size == %u)\n", index, size);
errno = EINVAL;
Expand All @@ -1173,7 +1175,9 @@ static inline int _lite3_set_by_append(unsigned char *buf, size_t *__restrict in
int ret;
if ((ret = _lite3_verify_arr_set(buf, inout_buflen, ofs, bufsz)) < 0)
return ret;
uint32_t size = (*(uint32_t *)(buf + ofs + LITE3_NODE_SIZE_KC_OFFSET)) >> LITE3_NODE_SIZE_SHIFT;
uint32_t size;
memcpy(&size, buf + ofs + LITE3_NODE_SIZE_KC_OFFSET, sizeof(size));
size >>= LITE3_NODE_SIZE_SHIFT;
lite3_key_data key_data = {
.hash = size,
.size = 0,
Expand Down Expand Up @@ -1728,7 +1732,9 @@ static inline int _lite3_get_by_index(const unsigned char *buf, size_t buflen, s
int ret;
if ((ret = _lite3_verify_arr_get(buf, buflen, ofs)) < 0)
return ret;
uint32_t size = (*(uint32_t *)(buf + ofs + LITE3_NODE_SIZE_KC_OFFSET)) >> LITE3_NODE_SIZE_SHIFT;
uint32_t size;
memcpy(&size, buf + ofs + LITE3_NODE_SIZE_KC_OFFSET, sizeof(size));
size >>= LITE3_NODE_SIZE_SHIFT;
if (LITE3_UNLIKELY(index >= size)) {
LITE3_PRINT_ERROR("INVALID ARGUMENT: ARRAY INDEX %u OUT OF BOUNDS (size == %u)\n", index, size);
errno = EINVAL;
Expand Down Expand Up @@ -1894,7 +1900,8 @@ static inline int lite3_count(
errno = EINVAL;
return -1;
}
*out = (*(uint32_t *)(buf + ofs + LITE3_NODE_SIZE_KC_OFFSET)) >> LITE3_NODE_SIZE_SHIFT;
memcpy(out, buf + ofs + LITE3_NODE_SIZE_KC_OFFSET, sizeof(*out));
*out >>= LITE3_NODE_SIZE_SHIFT;
return ret;
}

Expand Down
8 changes: 6 additions & 2 deletions vendor/lite3/include/lite3_context_api.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -693,7 +693,9 @@ static inline int _lite3_ctx_set_by_index(lite3_ctx *ctx, size_t ofs, uint32_t i
int ret;
if ((ret = _lite3_verify_arr_set(ctx->buf, &ctx->buflen, ofs, ctx->bufsz)) < 0)
return ret;
uint32_t size = (*(uint32_t *)(ctx->buf + ofs + LITE3_NODE_SIZE_KC_OFFSET)) >> LITE3_NODE_SIZE_SHIFT;
uint32_t size;
memcpy(&size, ctx->buf + ofs + LITE3_NODE_SIZE_KC_OFFSET, sizeof(size));
size >>= LITE3_NODE_SIZE_SHIFT;
if (LITE3_UNLIKELY(index > size)) {
LITE3_PRINT_ERROR("INVALID ARGUMENT: ARRAY INDEX %u OUT OF BOUNDS (size == %u)\n", index, size);
errno = EINVAL;
Expand All @@ -719,7 +721,9 @@ static inline int _lite3_ctx_set_by_append(lite3_ctx *ctx, size_t ofs, size_t va
int ret;
if ((ret = _lite3_verify_arr_set(ctx->buf, &ctx->buflen, ofs, ctx->bufsz)) < 0)
return ret;
uint32_t size = (*(uint32_t *)(ctx->buf + ofs + LITE3_NODE_SIZE_KC_OFFSET)) >> LITE3_NODE_SIZE_SHIFT;
uint32_t size;
memcpy(&size, ctx->buf + ofs + LITE3_NODE_SIZE_KC_OFFSET, sizeof(size));
size >>= LITE3_NODE_SIZE_SHIFT;
lite3_key_data key_data = {
.hash = size,
.size = 0,
Expand Down
12 changes: 12 additions & 0 deletions vendor/lite3/src/json_dec.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -72,11 +72,13 @@ int _lite3_json_dec_obj_switch(unsigned char *buf, size_t *restrict inout_buflen
case YYJSON_TYPE_NUM:
switch (yyjson_get_subtype(yy_val)) {
case YYJSON_SUBTYPE_SINT:
;
int64_t num_i64 = yyjson_get_sint(yy_val);
if ((ret = lite3_set_i64(buf, inout_buflen, ofs, bufsz, key, num_i64)) < 0)
return ret;
break;
case YYJSON_SUBTYPE_UINT:
;
uint64_t num_u64 = yyjson_get_uint(yy_val);
if (num_u64 <= INT64_MAX) {
if ((ret = lite3_set_i64(buf, inout_buflen, ofs, bufsz, key, (int64_t)num_u64)) < 0)
Expand All @@ -88,6 +90,7 @@ int _lite3_json_dec_obj_switch(unsigned char *buf, size_t *restrict inout_buflen
break;
}
case YYJSON_SUBTYPE_REAL:
;
double num_f64 = yyjson_get_real(yy_val);
if ((ret = lite3_set_f64(buf, inout_buflen, ofs, bufsz, key, num_f64)) < 0)
return ret;
Expand All @@ -99,19 +102,22 @@ int _lite3_json_dec_obj_switch(unsigned char *buf, size_t *restrict inout_buflen
}
break;
case YYJSON_TYPE_STR:
;
const char *str = yyjson_get_str(yy_val);
size_t len = yyjson_get_len(yy_val);
if ((ret = lite3_set_str_n(buf, inout_buflen, ofs, bufsz, key, str, len)) < 0)
return ret;
break;
case YYJSON_TYPE_OBJ:
;
size_t obj_ofs;
if ((ret = lite3_set_obj(buf, inout_buflen, ofs, bufsz, key, &obj_ofs)) < 0)
return ret;
if ((ret = _lite3_json_dec_obj(buf, inout_buflen, obj_ofs, bufsz, nesting_depth, doc, yy_val)) < 0)
return ret;
break;
case YYJSON_TYPE_ARR:
;
size_t arr_ofs;
if ((ret = lite3_set_arr(buf, inout_buflen, ofs, bufsz, key, &arr_ofs)) < 0)
return ret;
Expand Down Expand Up @@ -154,11 +160,13 @@ int _lite3_json_dec_arr_switch(unsigned char *buf, size_t *restrict inout_buflen
case YYJSON_TYPE_NUM:
switch (yyjson_get_subtype(yy_val)) {
case YYJSON_SUBTYPE_SINT:
;
int64_t num_i64 = yyjson_get_sint(yy_val);
if ((ret = lite3_arr_append_i64(buf, inout_buflen, ofs, bufsz, num_i64)) < 0)
return ret;
break;
case YYJSON_SUBTYPE_UINT:
;
uint64_t num_u64 = yyjson_get_uint(yy_val);
if (num_u64 <= INT64_MAX) {
if ((ret = lite3_arr_append_i64(buf, inout_buflen, ofs, bufsz, (int64_t)num_u64)) < 0)
Expand All @@ -170,6 +178,7 @@ int _lite3_json_dec_arr_switch(unsigned char *buf, size_t *restrict inout_buflen
break;
}
case YYJSON_SUBTYPE_REAL:
;
double num_f64 = yyjson_get_real(yy_val);
if ((ret = lite3_arr_append_f64(buf, inout_buflen, ofs, bufsz, num_f64)) < 0)
return ret;
Expand All @@ -181,19 +190,22 @@ int _lite3_json_dec_arr_switch(unsigned char *buf, size_t *restrict inout_buflen
}
break;
case YYJSON_TYPE_STR:
;
const char *str = yyjson_get_str(yy_val);
size_t len = yyjson_get_len(yy_val);
if ((ret = lite3_arr_append_str_n(buf, inout_buflen, ofs, bufsz, str, len)) < 0)
return ret;
break;
case YYJSON_TYPE_OBJ:
;
size_t obj_ofs;
if ((ret = lite3_arr_append_obj(buf, inout_buflen, ofs, bufsz, &obj_ofs)) < 0)
return ret;
if ((ret = _lite3_json_dec_obj(buf, inout_buflen, obj_ofs, bufsz, nesting_depth, doc, yy_val)) < 0)
return ret;
break;
case YYJSON_TYPE_ARR:
;
size_t arr_ofs;
if ((ret = lite3_arr_append_arr(buf, inout_buflen, ofs, bufsz, &arr_ofs)) < 0)
return ret;
Expand Down
2 changes: 2 additions & 0 deletions vendor/lite3/src/json_enc.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,7 @@ int _lite3_json_enc_switch(const unsigned char *buf, size_t buflen, size_t nesti
*yy_val = yyjson_mut_double(doc, lite3_val_f64(val));
break;
case LITE3_TYPE_BYTES:
;
size_t bytes_len;
const u8 *bytes = lite3_val_bytes(val, &bytes_len);
int b64_len;
Expand All @@ -98,6 +99,7 @@ int _lite3_json_enc_switch(const unsigned char *buf, size_t buflen, size_t nesti
free(b64);
break;
case LITE3_TYPE_STRING:
;
size_t str_len;
const char *str = lite3_val_str_n(val, &str_len);
/*
Expand Down
56 changes: 31 additions & 25 deletions vendor/lite3/src/lite3.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -131,12 +131,18 @@ static inline int _verify_key(
size_t *restrict inout_ofs, // key entry offset (relative to *buf)
size_t *restrict out_key_tag_size) // key tag size (optionally call with NULL)
{
if (LITE3_UNLIKELY(LITE3_KEY_TAG_SIZE_MAX > buflen || *inout_ofs > buflen - LITE3_KEY_TAG_SIZE_MAX)) {
if (LITE3_UNLIKELY(LITE3_KEY_TAG_SIZE_MIN > buflen || *inout_ofs > buflen - LITE3_KEY_TAG_SIZE_MIN)) {
LITE3_PRINT_ERROR("KEY ENTRY OUT OF BOUNDS\n");
errno = EFAULT;
return -1;
}
size_t _key_tag_size = (size_t)((*((u8 *)(buf + *inout_ofs)) & LITE3_KEY_TAG_SIZE_MASK) + 1);

if (LITE3_UNLIKELY(_key_tag_size > buflen || *inout_ofs > buflen - _key_tag_size)) {
LITE3_PRINT_ERROR("KEY ENTRY OUT OF BOUNDS\n");
errno = EFAULT;
return -1;
}
if (key_tag_size) {
if (key_tag_size != _key_tag_size) {
LITE3_PRINT_ERROR("KEY TAG SIZE DOES NOT MATCH\n");
Expand Down Expand Up @@ -243,9 +249,7 @@ int lite3_get_impl(

lite3_key_data attempt_key = key_data;
attempt_key.hash = key_data.hash + attempt * attempt;
#ifdef LITE3_DEBUG
LITE3_PRINT_DEBUG("probe attempt: %u\thash: %u\n", attempt, attempt_key.hash);
#endif
// LITE3_PRINT_DEBUG("probe attempt: %u\thash: %u\n", attempt, attempt_key.hash);

struct node *restrict node = __builtin_assume_aligned((struct node *)(buf + ofs), LITE3_NODE_ALIGNMENT);

Expand Down Expand Up @@ -394,16 +398,18 @@ int lite3_iter_next(const unsigned char *buf, size_t buflen, lite3_iter *iter, l
size_t target_ofs = node->kv_ofs[iter->node_i[iter->depth]];

int ret;
if (type == LITE3_TYPE_OBJECT && out_key) { // write back key if not NULL
if (type == LITE3_TYPE_OBJECT) { // write back key if not NULL
size_t key_tag_size;
size_t key_start_ofs = target_ofs;
if ((ret = _verify_key(buf, buflen, NULL, 0, 0, &target_ofs, &key_tag_size)) < 0)
return ret;
out_key->gen = iter->gen;
out_key->len = 0;
memcpy(&out_key->len, buf + key_start_ofs, key_tag_size);
--out_key->len; // Lite³ stores string size including NULL-terminator. Correction required for public API.
out_key->ptr = (const char *)(buf + key_start_ofs + key_tag_size);
if (out_key) {
out_key->gen = iter->gen;
out_key->len = 0;
memcpy(&out_key->len, buf + key_start_ofs, key_tag_size);
--out_key->len; // Lite³ stores string size including NULL-terminator. Correction required for public API.
out_key->ptr = (const char *)(buf + key_start_ofs + key_tag_size);
}
}
if (out_val_ofs) { // write back val if not NULL
size_t val_start_ofs = target_ofs;
Expand Down Expand Up @@ -447,19 +453,19 @@ int lite3_iter_next(const unsigned char *buf, size_t buflen, lite3_iter *iter, l
return -1;
}
#ifdef LITE3_PREFETCHING
__builtin_prefetch(buf + node->child_ofs[(iter->node_i[iter->depth] + 1) & LITE3_NODE_KEY_COUNT_MASK], 0, 2); // prefetch next nodes
__builtin_prefetch(buf + node->child_ofs[(iter->node_i[iter->depth] + 1) & LITE3_NODE_KEY_COUNT_MASK] + 64, 0, 2);
__builtin_prefetch(buf + node->child_ofs[(iter->node_i[iter->depth] + 2) & LITE3_NODE_KEY_COUNT_MASK], 0, 2);
__builtin_prefetch(buf + node->child_ofs[(iter->node_i[iter->depth] + 2) & LITE3_NODE_KEY_COUNT_MASK] + 64, 0, 2);
__builtin_prefetch(buf + node->child_ofs[(u32)(iter->node_i[iter->depth] + 1) & LITE3_NODE_KEY_COUNT_MASK], 0, 2); // prefetch next nodes
__builtin_prefetch(buf + node->child_ofs[(u32)(iter->node_i[iter->depth] + 1) & LITE3_NODE_KEY_COUNT_MASK] + 64, 0, 2);
__builtin_prefetch(buf + node->child_ofs[(u32)(iter->node_i[iter->depth] + 2) & LITE3_NODE_KEY_COUNT_MASK], 0, 2);
__builtin_prefetch(buf + node->child_ofs[(u32)(iter->node_i[iter->depth] + 2) & LITE3_NODE_KEY_COUNT_MASK] + 64, 0, 2);
#endif
}
#ifdef LITE3_PREFETCHING
__builtin_prefetch(buf + node->kv_ofs[(iter->node_i[iter->depth] + 0) & LITE3_NODE_KEY_COUNT_MASK], 0, 0); // prefetch next items
__builtin_prefetch(buf + node->kv_ofs[(iter->node_i[iter->depth] + 0) & LITE3_NODE_KEY_COUNT_MASK] + 64, 0, 0);
__builtin_prefetch(buf + node->kv_ofs[(iter->node_i[iter->depth] + 1) & LITE3_NODE_KEY_COUNT_MASK], 0, 0);
__builtin_prefetch(buf + node->kv_ofs[(iter->node_i[iter->depth] + 1) & LITE3_NODE_KEY_COUNT_MASK] + 64, 0, 0);
__builtin_prefetch(buf + node->kv_ofs[(iter->node_i[iter->depth] + 2) & LITE3_NODE_KEY_COUNT_MASK], 0, 0);
__builtin_prefetch(buf + node->kv_ofs[(iter->node_i[iter->depth] + 2) & LITE3_NODE_KEY_COUNT_MASK] + 64, 0, 0);
__builtin_prefetch(buf + node->kv_ofs[(u32)(iter->node_i[iter->depth] + 0) & LITE3_NODE_KEY_COUNT_MASK], 0, 0); // prefetch next items
__builtin_prefetch(buf + node->kv_ofs[(u32)(iter->node_i[iter->depth] + 0) & LITE3_NODE_KEY_COUNT_MASK] + 64, 0, 0);
__builtin_prefetch(buf + node->kv_ofs[(u32)(iter->node_i[iter->depth] + 1) & LITE3_NODE_KEY_COUNT_MASK], 0, 0);
__builtin_prefetch(buf + node->kv_ofs[(u32)(iter->node_i[iter->depth] + 1) & LITE3_NODE_KEY_COUNT_MASK] + 64, 0, 0);
__builtin_prefetch(buf + node->kv_ofs[(u32)(iter->node_i[iter->depth] + 2) & LITE3_NODE_KEY_COUNT_MASK], 0, 0);
__builtin_prefetch(buf + node->kv_ofs[(u32)(iter->node_i[iter->depth] + 2) & LITE3_NODE_KEY_COUNT_MASK] + 64, 0, 0);
#endif
return LITE3_ITER_ITEM;
}
Expand Down Expand Up @@ -556,9 +562,8 @@ int lite3_set_impl(

lite3_key_data attempt_key = key_data;
attempt_key.hash = key_data.hash + attempt * attempt;
#ifdef LITE3_DEBUG
LITE3_PRINT_DEBUG("probe attempt: %u\thash: %u\n", attempt, attempt_key.hash);
#endif
// LITE3_PRINT_DEBUG("probe attempt: %u\thash: %u\n", attempt, attempt_key.hash);


size_t entry_size = base_entry_size;
struct node *restrict parent = NULL;
Expand Down Expand Up @@ -672,10 +677,11 @@ int lite3_set_impl(
while (i < key_count && node->hashes[i] < attempt_key.hash)
i++;

LITE3_PRINT_DEBUG("i: %i\tkc: %i\tnode->hashes[i]: %u\n", i, key_count, node->hashes[i]);
// LITE3_PRINT_DEBUG("i: %i\tkc: %i\tnode->hashes[i]: %u\n", i, key_count, node->hashes[i]);

if (i < key_count && node->hashes[i] == attempt_key.hash) { // matching key found, already exists?
key_match_skip:
;
size_t target_ofs = node->kv_ofs[i];
size_t key_start_ofs = target_ofs;
if (key) {
Expand Down Expand Up @@ -752,7 +758,7 @@ int lite3_set_impl(
node->hashes[j] = node->hashes[j - 1];
node->kv_ofs[j] = node->kv_ofs[j - 1];
}
LITE3_PRINT_DEBUG("INSERTING HASH: %u\ti: %i\n", attempt_key.hash, i);
// LITE3_PRINT_DEBUG("INSERTING HASH: %u\ti: %i\n", attempt_key.hash, i);
node->hashes[i] = attempt_key.hash;
node->size_kc = (node->size_kc & ~LITE3_NODE_KEY_COUNT_MASK)
| ((node->size_kc + 1) & LITE3_NODE_KEY_COUNT_MASK); // key_count++
Expand Down