From 3703558a34ca1526148240d8c844ce7aa5b9d060 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miko=C5=82aj=20Paraniak?= Date: Fri, 6 Oct 2023 13:47:56 +0800 Subject: [PATCH 01/12] jets: implement zlib expansion --- pkg/noun/jets/e/zlib.c | 142 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 142 insertions(+) create mode 100644 pkg/noun/jets/e/zlib.c diff --git a/pkg/noun/jets/e/zlib.c b/pkg/noun/jets/e/zlib.c new file mode 100644 index 0000000000..7e3b81e0d3 --- /dev/null +++ b/pkg/noun/jets/e/zlib.c @@ -0,0 +1,142 @@ +/* +* DEFLATE decompression +*/ + +#include "jets/q.h" +#include "jets/w.h" +#include "manage.h" +#include "retrieve.h" + +#include "noun.h" + +#include + +#define OUTBUF_SZ 4096 + +u3_noun u3qe_zlib_expand(u3_atom pos, u3_noun byts) { + + c3_d pos_d; + c3_d wid_d; + + u3_atom wid; + u3_atom dat; + + u3x_mean(byts, 2, &wid, 3, &dat, 0); + + size_t sad_i = u3r_met(3, dat); + + if (c3n == (u3r_safe_chub(pos, &pos_d))) { + return u3_none; + } + if (c3n == (u3r_safe_chub(wid, &wid_d))) { + return u3_none; + } + + + c3_y* byt_y; + + if ( c3y == u3a_is_cat(dat)) { + byt_y = (c3_y*)&dat; + } + else { + u3a_atom* vat_u = u3a_to_ptr(dat); + byt_y = (c3_y*)vat_u->buf_w; + } + + c3_y* buf_y = byt_y; + + /* Leading zeros do not make sense + * for a Zlib stream + */ + u3_assert(wid_d == sad_i); + + z_stream zea; + c3_w zas_w; + + zea.next_in = buf_y + pos_d; + zea.avail_in = (sad_i - pos_d); + zea.zalloc = Z_NULL; + zea.zfree = Z_NULL; + + /* Allocate output buffer + */ + size_t sob_i = OUTBUF_SZ; + c3_y* cuf_y = c3_malloc(sob_i); + + u3_assert(cuf_y != NULL); + + zea.next_out = cuf_y; + zea.avail_out = sob_i; + + zas_w = inflateInit(&zea); + + if( Z_OK != zas_w) { + fprintf(stderr, "u3qe_zlib_expand: error while initializing Zlib, zas_w = %d\r\n", zas_w); + + c3_free(cuf_y); + return u3_none; + } + + while (Z_OK == (zas_w = inflate(&zea, Z_FINISH))) { + + if (zea.avail_in == 0) break; + + if (zea.avail_out == 0) { + + c3_y* new_y = c3_realloc(cuf_y, sob_i + OUTBUF_SZ); + u3_assert(new_y != NULL); + + cuf_y = new_y; + + zea.avail_out = OUTBUF_SZ; + zea.next_out = cuf_y + sob_i; + + sob_i += OUTBUF_SZ; + } + } + + if (zas_w != Z_STREAM_END) { + fprintf(stderr, "u3qe_zlib_expand: error while expanding, zas_w = %d\r\n", zas_w); + + c3_free(cuf_y); + return u3_none; + } + + + + u3i_slab sab_u; + size_t len_i = sob_i - zea.avail_out; + u3i_slab_bare(&sab_u, 3, len_i); + // XX find out why this is important - otherwise we get memory leaks + // Are slabs supposed to be zero terminated at their length? + // + sab_u.buf_w[sab_u.len_w - 1] = 0; + memmove(sab_u.buf_y, cuf_y, len_i); + + u3_atom len_a = u3i_chub(len_i); + + pos_d += zea.total_in; + + zas_w = inflateEnd(&zea); + + if (zas_w != Z_OK) { + fprintf(stderr, "u3qe_zlib_expand: zlib stream inconsistent upon finish, zas_w = %d\r\n", + zas_w); + } + + c3_free(cuf_y); + // u3k is needed for byts, since it becomes a + // part of a new cell + return u3nc(u3nc(len_a, u3i_slab_moot_bytes(&sab_u)), + u3nc(u3i_chub(pos_d), u3k(byts))); +} + +u3_noun u3we_zlib_expand(u3_noun sea) { + + u3_atom pos; + u3_noun byts; + + u3x_mean(sea, u3x_sam_2, &pos, u3x_sam_3, &byts, 0); + + return u3qe_zlib_expand(pos, byts); +} From d1c49a4c9c50897ba23ca236cec8fdca2365fb61 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miko=C5=82aj=20Paraniak?= Date: Wed, 18 Oct 2023 11:41:23 +0800 Subject: [PATCH 02/12] jets: fix zlib expansion --- pkg/noun/jets/e/zlib.c | 7 +++++-- pkg/noun/jets/q.h | 2 ++ pkg/noun/jets/tree.c | 11 +++++++++++ pkg/noun/jets/w.h | 2 ++ 4 files changed, 20 insertions(+), 2 deletions(-) diff --git a/pkg/noun/jets/e/zlib.c b/pkg/noun/jets/e/zlib.c index 7e3b81e0d3..a5d5f103a3 100644 --- a/pkg/noun/jets/e/zlib.c +++ b/pkg/noun/jets/e/zlib.c @@ -77,7 +77,7 @@ u3_noun u3qe_zlib_expand(u3_atom pos, u3_noun byts) { return u3_none; } - while (Z_OK == (zas_w = inflate(&zea, Z_FINISH))) { + while (Z_OK == (zas_w = inflate(&zea, Z_FINISH)) || zas_w == Z_BUF_ERROR) { if (zea.avail_in == 0) break; @@ -96,7 +96,7 @@ u3_noun u3qe_zlib_expand(u3_atom pos, u3_noun byts) { } if (zas_w != Z_STREAM_END) { - fprintf(stderr, "u3qe_zlib_expand: error while expanding, zas_w = %d\r\n", zas_w); + fprintf(stderr, "u3qe_zlib_expand: error while expanding, zas_w = %d, msg = %s\r\n", zas_w, zea.msg); c3_free(cuf_y); return u3_none; @@ -122,6 +122,9 @@ u3_noun u3qe_zlib_expand(u3_atom pos, u3_noun byts) { if (zas_w != Z_OK) { fprintf(stderr, "u3qe_zlib_expand: zlib stream inconsistent upon finish, zas_w = %d\r\n", zas_w); + + c3_free(cuf_y); + return u3_none; } c3_free(cuf_y); diff --git a/pkg/noun/jets/q.h b/pkg/noun/jets/q.h index fd7c5981fb..f813c098f9 100644 --- a/pkg/noun/jets/q.h +++ b/pkg/noun/jets/q.h @@ -159,6 +159,8 @@ u3_noun u3qe_json_de(u3_atom); u3_atom u3qe_json_en(u3_noun); + u3_noun u3qe_zlib_expand(u3_atom, u3_noun); + u3_noun u3qeo_raw(u3_atom, u3_atom); u3_noun u3qef_drg(u3_noun, u3_atom); diff --git a/pkg/noun/jets/tree.c b/pkg/noun/jets/tree.c index e40fe03846..9b09a817b3 100644 --- a/pkg/noun/jets/tree.c +++ b/pkg/noun/jets/tree.c @@ -2120,6 +2120,16 @@ static u3j_core _139_hex_json_d[] = {} }; +/* + * DEFLATE decompression + */ +static u3j_harm _139_hex__zlib_expand_a[] = {{".2", u3we_zlib_expand, c3y}, {}}; +static u3j_core _139_hex__zlib_d[] = + { + { "expand", 7, _139_hex__zlib_expand_a, 0, no_hashes }, + {} + }; + static u3j_core _139_hex_d[] = { { "lore", 63, _140_hex_lore_a, 0, no_hashes }, { "leer", 63, _140_hex_leer_a, 0, no_hashes }, @@ -2138,6 +2148,7 @@ static u3j_core _139_hex_d[] = { "secp", 6, 0, _140_hex_secp_d, no_hashes }, { "mimes", 31, 0, _140_hex_mimes_d, no_hashes }, { "json", 31, 0, _139_hex_json_d, no_hashes }, + { "zlib", 31, 0, _139_hex__zlib_d, no_hashes}, {} }; diff --git a/pkg/noun/jets/w.h b/pkg/noun/jets/w.h index 34716971bd..7cf541ee63 100644 --- a/pkg/noun/jets/w.h +++ b/pkg/noun/jets/w.h @@ -226,6 +226,8 @@ u3_noun u3we_json_de(u3_noun); u3_atom u3we_json_en(u3_noun); + u3_noun u3we_zlib_expand(u3_noun); + u3_noun u3we_bend_fun(u3_noun); u3_noun u3we_cold_fun(u3_noun); u3_noun u3we_cook_fun(u3_noun); From 90707a0d50293a8d2390027ae3395779922e8c3a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miko=C5=82aj=20Paraniak?= Date: Wed, 25 Oct 2023 11:55:32 +0800 Subject: [PATCH 03/12] Add zlib to dependecies --- pkg/noun/BUILD.bazel | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/noun/BUILD.bazel b/pkg/noun/BUILD.bazel index a6b8de6d7a..fd6963cb82 100644 --- a/pkg/noun/BUILD.bazel +++ b/pkg/noun/BUILD.bazel @@ -40,6 +40,7 @@ vere_library( "@sigsegv", "@softfloat", "@urcrypt", + "@zlib", ] + select({ "@platforms//os:macos": ["//pkg/noun/platform/darwin"], "@platforms//os:linux": ["//pkg/noun/platform/linux"], From 144f08dccb9f5430cc6fda546d8b1927b3644398 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miko=C5=82aj=20Paraniak?= Date: Wed, 6 Dec 2023 13:15:59 +0100 Subject: [PATCH 04/12] jets: add libstream jets, add libgit jets --- pkg/noun/jets/e/git.c | 181 +++++++++++++++++++++++++ pkg/noun/jets/e/git_protocol.c | 182 +++++++++++++++++++++++++ pkg/noun/jets/e/stream.c | 241 +++++++++++++++++++++++++++++++++ pkg/noun/jets/e/zlib.c | 75 +++++++--- pkg/noun/jets/q.h | 6 + pkg/noun/jets/tree.c | 46 ++++++- pkg/noun/jets/w.h | 6 + 7 files changed, 712 insertions(+), 25 deletions(-) create mode 100644 pkg/noun/jets/e/git.c create mode 100644 pkg/noun/jets/e/git_protocol.c create mode 100644 pkg/noun/jets/e/stream.c diff --git a/pkg/noun/jets/e/git.c b/pkg/noun/jets/e/git.c new file mode 100644 index 0000000000..2acb6b1ca3 --- /dev/null +++ b/pkg/noun/jets/e/git.c @@ -0,0 +1,181 @@ +#include "c3.h" + +#include "jets/q.h" +#include "jets/w.h" + +#include "allocate.h" +#include "noun.h" + +#define FLUSH_PKT 0 +#define DELIM_PKT 1 +#define END_PKT 2 + +#define PKT_LINE_MAX ((1 << 16) - 1 - 4) + +static c3_w _hex_dit(c3_y byt) +{ + // XX parse in the main function + // + u3_assert(('0' <= byt && byt <= '9') || ('a' <= byt && byt <= 'f')); + + c3_w dit_w; + + if (byt > '9') { + dit_w = (byt - 'a') + 10; + } + else { + dit_w = (byt - '0'); + } + + return dit_w; +} + +u3_noun u3qe_git_protocol_stream_pkt_lines_on_band( + u3_atom band, + u3_noun sea){ + + if (c3n == u3a_is_cat(band)) { + u3m_bail(c3__fail); + } + + c3_w band_w; + + if (c3y != u3r_safe_word(band, &band_w)){ + u3m_bail(c3__fail); + } + + u3_atom pos, p_octs, q_octs; + u3x_mean(sea, 2, &pos, 6, &p_octs, 7, &q_octs, 0); + + c3_w pos_w, p_octs_w; + + if (c3n == u3r_safe_word(pos, &pos_w)) { + u3m_bail(c3__fail); + } + + if (c3n == u3r_safe_word(p_octs, &p_octs_w)) { + u3m_bail(c3__fail); + } + + if (c3n == u3a_is_atom(q_octs)) { + u3m_bail(c3__fail); + } + + c3_y *sea_y; + + if (c3y == u3a_is_cat(q_octs)) { + sea_y = (c3_y*)&q_octs; + } + else { + u3a_atom *a = u3a_to_ptr(q_octs); + // Verify octs sanity + // + u3_assert(p_octs_w == u3r_met(3, q_octs)); + sea_y = (c3_y*)a->buf_w; + } + + sea_y += pos_w; + + c3_w len_w = PKT_LINE_MAX; + c3_w total_w = 0; + c3_y *buf_y = c3_malloc(len_w); + + if (buf_y == NULL) { + u3m_bail(c3__fail); + } + + c3_w pkt_len_w; + c3_y pkt_band_y; + + while (pos_w < p_octs_w) { + + /* + * Parse pkt-line length + */ + if (pos_w + 4 > p_octs_w) { + c3_free(buf_y); + fprintf(stderr, "u3qe_git__stream_pkt_lines_on_band: parsing failure\r\n"); + return u3_none; + } + + pkt_len_w = (_hex_dit(sea_y[0]) << 12) + + (_hex_dit(sea_y[1]) << 8) + + (_hex_dit(sea_y[2]) << 4) + + _hex_dit(sea_y[3]); + // fprintf(stderr, "u3qe_git__stream_pkt_lines_on_band: pos_w = %d, pkt_len_w = %d, sea_y = %.10s\r\n", pos_w, pkt_len_w, sea_y); + + if (FLUSH_PKT == pkt_len_w) { + break; + } + + pos_w += 4; + sea_y += 4; + pkt_len_w -= 4; + + u3_assert(pkt_len_w <= PKT_LINE_MAX); + + /* + * Make space for pkt_len_w bytes in the buffer + */ + if (pos_w + pkt_len_w > p_octs_w) { + c3_free(buf_y); + u3m_bail(c3__fail); + } + + if (total_w + pkt_len_w > len_w) { + + // Should we malloc & copy here? + // + c3_y *new_y = c3_malloc(len_w + 2*PKT_LINE_MAX); + + if ( new_y == NULL) { + c3_free(buf_y); + u3m_bail(c3__fail); + } + + memcpy(new_y, buf_y, len_w); + len_w += 2*PKT_LINE_MAX; + + c3_free(buf_y); + buf_y = new_y; + } + + /* + * Parse band + */ + pkt_band_y = *sea_y; + + sea_y += 1; + pos_w += 1; + pkt_len_w -= 1; + + memmove(buf_y + total_w, sea_y, pkt_len_w); + + if (pkt_band_y == band_w) { + // fprintf(stderr, "u3we_git__stream_pkt_lines_on_band: accumulating %d bytes\r\n", pkt_len_w); + total_w += pkt_len_w; + } + else { + // buf_y[total_w + pkt_len_w] = 0; + // fprintf(stderr, "[band %d] %s\r\n", pkt_band_y, buf_y + total_w); + } + + sea_y += pkt_len_w; + pos_w += pkt_len_w; + } + + u3_noun octs_red = u3nc(u3i_word(total_w), u3i_bytes(total_w, buf_y)); + c3_free(buf_y); + + return u3nc(u3nc(u3i_word(0), octs_red), u3k(sea)); +} +u3_noun u3we_git_pak_expand_delta_object(u3_noun cor) { + + u3_atom band; + u3_noun sea; + + u3x_mean(cor, u3x_sam_2, &band, + u3x_sam_3, &sea, 0); + + return u3qe_git_protocol_stream_pkt_lines_on_band(band, sea); +} diff --git a/pkg/noun/jets/e/git_protocol.c b/pkg/noun/jets/e/git_protocol.c new file mode 100644 index 0000000000..3b0fe5c1da --- /dev/null +++ b/pkg/noun/jets/e/git_protocol.c @@ -0,0 +1,182 @@ + +#include "c3.h" + +#include "jets/q.h" +#include "jets/w.h" + +#include "allocate.h" +#include "noun.h" + +#define FLUSH_PKT 0 +#define DELIM_PKT 1 +#define END_PKT 2 + +#define PKT_LINE_MAX ((1 << 16) - 1 - 4) + +static c3_w _hex_dit(c3_y byt) +{ + // XX parse in the main function + // + u3_assert(('0' <= byt && byt <= '9') || ('a' <= byt && byt <= 'f')); + + c3_w dit_w; + + if (byt > '9') { + dit_w = (byt - 'a') + 10; + } + else { + dit_w = (byt - '0'); + } + + return dit_w; +} + +u3_noun u3qe_git_protocol_stream_pkt_lines_on_band( + u3_atom band, + u3_noun sea){ + + if (c3n == u3a_is_cat(band)) { + u3m_bail(c3__fail); + } + + c3_w band_w; + + if (c3y != u3r_safe_word(band, &band_w)){ + u3m_bail(c3__fail); + } + + u3_atom pos, p_octs, q_octs; + u3x_mean(sea, 2, &pos, 6, &p_octs, 7, &q_octs, 0); + + c3_w pos_w, p_octs_w; + + if (c3n == u3r_safe_word(pos, &pos_w)) { + u3m_bail(c3__fail); + } + + if (c3n == u3r_safe_word(p_octs, &p_octs_w)) { + u3m_bail(c3__fail); + } + + if (c3n == u3a_is_atom(q_octs)) { + u3m_bail(c3__fail); + } + + c3_y *sea_y; + + if (c3y == u3a_is_cat(q_octs)) { + sea_y = (c3_y*)&q_octs; + } + else { + u3a_atom *a = u3a_to_ptr(q_octs); + // Verify octs sanity + // + u3_assert(p_octs_w == u3r_met(3, q_octs)); + sea_y = (c3_y*)a->buf_w; + } + + sea_y += pos_w; + + c3_w len_w = PKT_LINE_MAX; + c3_w total_w = 0; + c3_y *buf_y = c3_malloc(len_w); + + if (buf_y == NULL) { + u3m_bail(c3__fail); + } + + c3_w pkt_len_w; + c3_y pkt_band_y; + + while (pos_w < p_octs_w) { + + /* + * Parse pkt-line length + */ + if (pos_w + 4 > p_octs_w) { + c3_free(buf_y); + fprintf(stderr, "u3qe_git__stream_pkt_lines_on_band: parsing failure\r\n"); + return u3_none; + } + + pkt_len_w = (_hex_dit(sea_y[0]) << 12) + + (_hex_dit(sea_y[1]) << 8) + + (_hex_dit(sea_y[2]) << 4) + + _hex_dit(sea_y[3]); + // fprintf(stderr, "u3qe_git__stream_pkt_lines_on_band: pos_w = %d, pkt_len_w = %d, sea_y = %.10s\r\n", pos_w, pkt_len_w, sea_y); + + if (FLUSH_PKT == pkt_len_w) { + break; + } + + pos_w += 4; + sea_y += 4; + pkt_len_w -= 4; + + u3_assert(pkt_len_w <= PKT_LINE_MAX); + + /* + * Make space for pkt_len_w bytes in the buffer + */ + if (pos_w + pkt_len_w > p_octs_w) { + c3_free(buf_y); + u3m_bail(c3__fail); + } + + if (total_w + pkt_len_w > len_w) { + + // Should we malloc & copy here? + // + c3_y *new_y = c3_malloc(len_w + 2*PKT_LINE_MAX); + + if ( new_y == NULL) { + c3_free(buf_y); + u3m_bail(c3__fail); + } + + memcpy(new_y, buf_y, len_w); + len_w += 2*PKT_LINE_MAX; + + c3_free(buf_y); + buf_y = new_y; + } + + /* + * Parse band + */ + pkt_band_y = *sea_y; + + sea_y += 1; + pos_w += 1; + pkt_len_w -= 1; + + memmove(buf_y + total_w, sea_y, pkt_len_w); + + if (pkt_band_y == band_w) { + // fprintf(stderr, "u3we_git__stream_pkt_lines_on_band: accumulating %d bytes\r\n", pkt_len_w); + total_w += pkt_len_w; + } + else { + // buf_y[total_w + pkt_len_w] = 0; + // fprintf(stderr, "[band %d] %s\r\n", pkt_band_y, buf_y + total_w); + } + + sea_y += pkt_len_w; + pos_w += pkt_len_w; + } + + u3_noun octs_red = u3nc(u3i_word(total_w), u3i_bytes(total_w, buf_y)); + c3_free(buf_y); + + return u3nc(u3nc(u3i_word(0), octs_red), u3k(sea)); +} +u3_noun u3we_git_protocol_stream_pkt_lines_on_band(u3_noun cor) { + + u3_atom band; + u3_noun sea; + + u3x_mean(cor, u3x_sam_2, &band, + u3x_sam_3, &sea, 0); + + return u3qe_git_protocol_stream_pkt_lines_on_band(band, sea); +} diff --git a/pkg/noun/jets/e/stream.c b/pkg/noun/jets/e/stream.c new file mode 100644 index 0000000000..f3cc9311ba --- /dev/null +++ b/pkg/noun/jets/e/stream.c @@ -0,0 +1,241 @@ +#include "jets/q.h" +#include "jets/w.h" + +#include "allocate.h" +#include "noun.h" + + +u3_noun u3qe_stream_append_get_bytes(u3_atom n, u3_noun red, u3_noun sea) +{ + u3_atom pos_red; + u3_atom p_octs_red, q_octs_red; + + u3_atom pos_sea; + u3_atom p_octs_sea, q_octs_sea; + + u3x_mean(red, 2, &pos_red, + 6, &p_octs_red, + 7, &q_octs_red, 0); + + u3x_mean(sea, 2, &pos_sea, + 6, &p_octs_sea, + 7, &q_octs_sea, 0); + + if (c3y == u3a_is_cat(n) && n == 0) { + return u3nc(u3k(red), u3k(sea)); + } + c3_w pos_red_w, pos_sea_w; + + if (c3y != u3r_safe_word(pos_red, &pos_red_w)) { + return u3_none; + } + + if (c3y != u3r_safe_word(pos_sea, &pos_sea_w)) { + return u3_none; + } + + c3_w n_w; + + if (c3y != u3r_safe_word(n, &n_w)) { + return u3_none; + } + + c3_y *red_buf_y, *sea_buf_y; + + if (c3y == u3a_is_cat(q_octs_red)) { + red_buf_y = (c3_y*)&q_octs_red; + } + else { + u3a_atom *a = u3a_to_ptr(q_octs_red); + red_buf_y = (c3_y*)a->buf_w; + } + + if (c3y == u3a_is_cat(q_octs_sea)) { + sea_buf_y = (c3_y*)&q_octs_sea; + } + else { + u3a_atom *a = u3a_to_ptr(q_octs_sea); + sea_buf_y = (c3_y*)a->buf_w; + } + + c3_w p_octs_red_w, p_octs_sea_w; + + if (c3y != u3r_safe_word(p_octs_red, &p_octs_red_w)) { + return u3_none; + } + + if (c3y != u3r_safe_word(p_octs_sea, &p_octs_sea_w)) { + return u3_none; + } + // + // 1. Verify sea has n bytes from its current position + // 2. Get current addresses of red and sea + // 3. Resize red to accomodate n bytes more + // 4. Copy data from red to sea + // 5. Return and update size of red + // + if (pos_sea_w + n_w > p_octs_sea_w) { + fprintf(stderr, "u3qe_stream_append_get_bytes: sea exhausted\r\n"); + return u3_none; + } + + // XX is there a way to enlarge an existing storage? + + // Make space for expanded red + // + c3_i red_len_i = u3r_met(3, q_octs_red); + c3_i new_red_len_i = p_octs_red_w + n_w; + + c3_y *buf_y = c3_malloc(1+new_red_len_i); + buf_y[new_red_len_i] = 0; + + // Copy over existing content in red + // + memcpy(buf_y, red_buf_y, red_len_i); + + if (p_octs_red_w > red_len_i) { + memset(buf_y + red_len_i, 0, p_octs_red_w - red_len_i); + } + + // Copy n_w bytes from sea + // + memcpy(buf_y+p_octs_red_w, sea_buf_y + pos_sea_w, n_w); + + u3_atom new_red = u3nt(u3k(pos_red), + u3i_chub(new_red_len_i), + u3i_bytes(new_red_len_i, buf_y)); + + c3_free(buf_y); + + return u3nc(new_red, u3k(sea)); +} + +u3_noun u3we_stream_append_get_bytes(u3_noun cor){ + + u3_atom n; + u3_noun red, sea; + + u3x_mean(cor, u3x_sam_2, &n, + u3x_sam_6, &red, + u3x_sam_7, &sea, 0); + + return u3qe_stream_append_get_bytes(n, red, sea); +} + +u3_noun u3qe_stream_append_read_bytes(u3_atom n, u3_noun red, u3_noun sea) +{ + u3_atom pos_red; + u3_atom p_octs_red, q_octs_red; + + u3_atom pos_sea; + u3_noun octs_sea; + u3_atom p_octs_sea, q_octs_sea; + + u3x_mean(red, 2, &pos_red, + 6, &p_octs_red, + 7, &q_octs_red, 0); + + u3x_mean(sea, 2, &pos_sea, 3, &octs_sea, 0); + u3x_mean(octs_sea, 2, &p_octs_sea, 3, &q_octs_sea, 0); + + if (c3y == u3a_is_cat(n) && n == 0) { + return u3nc(u3k(red), u3k(sea)); + } + c3_w pos_red_w, pos_sea_w; + + if (c3y != u3r_safe_word(pos_red, &pos_red_w)) { + return u3_none; + } + + if (c3y != u3r_safe_word(pos_sea, &pos_sea_w)) { + return u3_none; + } + + c3_w n_w; + + if (c3y != u3r_safe_word(n, &n_w)) { + return u3_none; + } + + c3_y *red_buf_y, *sea_buf_y; + + if (c3y == u3a_is_cat(q_octs_red)) { + red_buf_y = (c3_y*)&q_octs_red; + } + else { + u3a_atom *a = u3a_to_ptr(q_octs_red); + red_buf_y = (c3_y*)a->buf_w; + } + + if (c3y == u3a_is_cat(q_octs_sea)) { + sea_buf_y = (c3_y*)&q_octs_sea; + } + else { + u3a_atom *a = u3a_to_ptr(q_octs_sea); + sea_buf_y = (c3_y*)a->buf_w; + } + + c3_w p_octs_red_w, p_octs_sea_w; + + if (c3y != u3r_safe_word(p_octs_red, &p_octs_red_w)) { + return u3_none; + } + + if (c3y != u3r_safe_word(p_octs_sea, &p_octs_sea_w)) { + return u3_none; + } + // + // 1. Verify sea has n bytes from its current position + // 2. Get current addresses of red and sea + // 3. Resize red to accomodate n bytes more + // 4. Copy data from red to sea + // 5. Return and update size of red + // + if (pos_sea_w + n_w > p_octs_sea_w) { + fprintf(stderr, "u3qe_stream_append_read_bytes: sea exhausted\r\n"); + return u3_none; + } + + // XX is there a way to enlarge an existing storage? + + // Make space for expanded red + // + c3_i red_len_i = u3r_met(3, q_octs_red); + c3_i new_red_len_i = p_octs_red_w + n_w; + + c3_y *buf_y = c3_malloc(1+new_red_len_i); + buf_y[new_red_len_i] = 0; + + // Copy over existing content in red + // + memcpy(buf_y, red_buf_y, red_len_i); + + if (p_octs_red_w > red_len_i) { + memset(buf_y + red_len_i, 0, p_octs_red_w - red_len_i); + } + + // Copy n_w bytes from sea + // + memcpy(buf_y+p_octs_red_w, sea_buf_y + pos_sea_w, n_w); + + u3_atom new_red = u3nt(u3k(pos_red), + u3i_chub(new_red_len_i), + u3i_bytes(new_red_len_i, buf_y)); + + c3_free(buf_y); + + return u3nc(new_red, u3nc(u3i_word(pos_sea_w + n_w), u3k(octs_sea))); +} + +u3_noun u3we_stream_append_read_bytes(u3_noun cor){ + + u3_atom n; + u3_noun red, sea; + + u3x_mean(cor, u3x_sam_2, &n, + u3x_sam_6, &red, + u3x_sam_7, &sea, 0); + + return u3qe_stream_append_read_bytes(n, red, sea); +} + diff --git a/pkg/noun/jets/e/zlib.c b/pkg/noun/jets/e/zlib.c index a5d5f103a3..da526655a0 100644 --- a/pkg/noun/jets/e/zlib.c +++ b/pkg/noun/jets/e/zlib.c @@ -10,6 +10,7 @@ #include "noun.h" #include +#include #define OUTBUF_SZ 4096 @@ -32,7 +33,6 @@ u3_noun u3qe_zlib_expand(u3_atom pos, u3_noun byts) { return u3_none; } - c3_y* byt_y; if ( c3y == u3a_is_cat(dat)) { @@ -43,8 +43,6 @@ u3_noun u3qe_zlib_expand(u3_atom pos, u3_noun byts) { byt_y = (c3_y*)vat_u->buf_w; } - c3_y* buf_y = byt_y; - /* Leading zeros do not make sense * for a Zlib stream */ @@ -53,13 +51,18 @@ u3_noun u3qe_zlib_expand(u3_atom pos, u3_noun byts) { z_stream zea; c3_w zas_w; - zea.next_in = buf_y + pos_d; - zea.avail_in = (sad_i - pos_d); + zea.next_in = byt_y + pos_d; + zea.avail_in = (wid_d - pos_d); + zea.zalloc = Z_NULL; - zea.zfree = Z_NULL; + zea.zfree = Z_NULL; + zea.opaque = Z_NULL; /* Allocate output buffer */ + + // Size of the output buffer + // size_t sob_i = OUTBUF_SZ; c3_y* cuf_y = c3_malloc(sob_i); @@ -79,7 +82,10 @@ u3_noun u3qe_zlib_expand(u3_atom pos, u3_noun byts) { while (Z_OK == (zas_w = inflate(&zea, Z_FINISH)) || zas_w == Z_BUF_ERROR) { - if (zea.avail_in == 0) break; + if (zea.avail_in == 0){ + fprintf(stderr, "Exhausted input\r\n"); + break; + } if (zea.avail_out == 0) { @@ -89,6 +95,9 @@ u3_noun u3qe_zlib_expand(u3_atom pos, u3_noun byts) { cuf_y = new_y; zea.avail_out = OUTBUF_SZ; + + u3_assert(sob_i == zea.total_out); + zea.next_out = cuf_y + sob_i; sob_i += OUTBUF_SZ; @@ -96,22 +105,50 @@ u3_noun u3qe_zlib_expand(u3_atom pos, u3_noun byts) { } if (zas_w != Z_STREAM_END) { + + fprintf(stderr, "u3qe_zlib_expand: %d input bytes\r\n", sad_i); + fprintf(stderr, "u3qe_zlib_expand: processed %d bytes\r\n", zea.total_in); + fprintf(stderr, "u3qe_zlib_expand: uncompressed %d bytes\r\n", zea.total_out); fprintf(stderr, "u3qe_zlib_expand: error while expanding, zas_w = %d, msg = %s\r\n", zas_w, zea.msg); - c3_free(cuf_y); - return u3_none; - } + // Dump ZLIB stream + // + time_t now; + time(&now); + char filename[256]; + if ( zas_w != Z_STREAM_END ) { + sprintf(filename, "error-stream-%d.dat", now); + } + else { + sprintf(filename, "good-stream-%d.dat", now); + } + + fprintf(stderr, "u3qe_zlib_expand: dumping corrupted stream to %s\r\n", filename); + FILE* file= fopen(filename, "w"); + assert(file != NULL); + // byt_y + pos_d, for zea.total_in bytes + // + fwrite(byt_y+pos_d, sizeof(char), zea.total_in, file); + + fclose(file); + + if ( zas_w != Z_STREAM_END ) { + c3_free(cuf_y); + return u3_none; + } + } - u3i_slab sab_u; size_t len_i = sob_i - zea.avail_out; - u3i_slab_bare(&sab_u, 3, len_i); - // XX find out why this is important - otherwise we get memory leaks - // Are slabs supposed to be zero terminated at their length? - // - sab_u.buf_w[sab_u.len_w - 1] = 0; - memmove(sab_u.buf_y, cuf_y, len_i); + c3_y *buf_y = c3_malloc(len_i); + + if (buf_y == NULL) { + c3_free(cuf_y); + u3m_bail(c3__meme); + } + + memcpy(buf_y, cuf_y, len_i); u3_atom len_a = u3i_chub(len_i); @@ -128,9 +165,7 @@ u3_noun u3qe_zlib_expand(u3_atom pos, u3_noun byts) { } c3_free(cuf_y); - // u3k is needed for byts, since it becomes a - // part of a new cell - return u3nc(u3nc(len_a, u3i_slab_moot_bytes(&sab_u)), + return u3nc(u3nc(len_a, u3i_bytes(len_i, buf_y)), u3nc(u3i_chub(pos_d), u3k(byts))); } diff --git a/pkg/noun/jets/q.h b/pkg/noun/jets/q.h index f813c098f9..05090b8bd1 100644 --- a/pkg/noun/jets/q.h +++ b/pkg/noun/jets/q.h @@ -161,6 +161,12 @@ u3_noun u3qe_zlib_expand(u3_atom, u3_noun); + u3_noun u3qe_stream_append_get_bytes(u3_atom n, u3_noun red, u3_noun sea); + u3_noun u3qe_stream_append_read_bytes(u3_atom n, u3_noun red, u3_noun sea); + + u3_noun u3qe_git_pak_expand_delta_object(u3_noun, u3_noun); + u3_noun u3qe_git_protocol_stream_pkt_lines_on_band(u3_atom band, u3_noun sea); + u3_noun u3qeo_raw(u3_atom, u3_atom); u3_noun u3qef_drg(u3_noun, u3_atom); diff --git a/pkg/noun/jets/tree.c b/pkg/noun/jets/tree.c index 9b09a817b3..319b6a6e61 100644 --- a/pkg/noun/jets/tree.c +++ b/pkg/noun/jets/tree.c @@ -2121,15 +2121,47 @@ static u3j_core _139_hex_json_d[] = }; /* - * DEFLATE decompression + * Zlib */ -static u3j_harm _139_hex__zlib_expand_a[] = {{".2", u3we_zlib_expand, c3y}, {}}; -static u3j_core _139_hex__zlib_d[] = +static u3j_harm _139_hex_zlib_expand_a[] = {{".2", u3we_zlib_expand, c3y}, {}}; +static u3j_core _139_hex_zlib_d[] = { - { "expand", 7, _139_hex__zlib_expand_a, 0, no_hashes }, + { "expand", 7, _139_hex_zlib_expand_a, 0, no_hashes }, {} }; +/* + * Stream library + */ +static u3j_harm _139_hex_stream_append_get_bytes_a[] = + {{".2", u3we_stream_append_get_bytes, c3y},{}}; + +static u3j_harm _139_hex_stream_append_read_bytes_a[] = + {{".2", u3we_stream_append_read_bytes, c3y},{}}; + +static u3j_core _139_hex_stream_d[] = + { + { "append-get-bytes", 7, _139_hex_stream_append_get_bytes_a, 0, no_hashes }, + { "append-read-bytes", 7, _139_hex_stream_append_read_bytes_a, 0, no_hashes }, + {} + }; + +/* + * Git protocol + */ +static u3j_harm _139_hex_git_protocol_stream_pkt_lines_on_band_a[] = + {{".2", u3we_git_protocol_stream_pkt_lines_on_band, c3y}, {}}; + +static u3j_core _139_hex_git_protocol_d[] = + { + { "stream-pkt-lines-on-band", 7, _139_hex_git_protocol_stream_pkt_lines_on_band_a, 0, no_hashes}, + {} + }; + +/* + * Git core + */ + static u3j_core _139_hex_d[] = { { "lore", 63, _140_hex_lore_a, 0, no_hashes }, { "leer", 63, _140_hex_leer_a, 0, no_hashes }, @@ -2148,7 +2180,11 @@ static u3j_core _139_hex_d[] = { "secp", 6, 0, _140_hex_secp_d, no_hashes }, { "mimes", 31, 0, _140_hex_mimes_d, no_hashes }, { "json", 31, 0, _139_hex_json_d, no_hashes }, - { "zlib", 31, 0, _139_hex__zlib_d, no_hashes}, + { "zlib", 31, 0, _139_hex_zlib_d, no_hashes}, + { "stream", 31, 0, _139_hex_stream_d, no_hashes}, + { "git", 31, 0, _139_hex_git_d, no_hashes}, + // XX this be under git, even though the cores are separate + { "git-protocol", 31, 0, _139_hex_git_protocol_d, no_hashes}, {} }; diff --git a/pkg/noun/jets/w.h b/pkg/noun/jets/w.h index 7cf541ee63..d95e847f50 100644 --- a/pkg/noun/jets/w.h +++ b/pkg/noun/jets/w.h @@ -228,6 +228,12 @@ u3_noun u3we_zlib_expand(u3_noun); + u3_noun u3we_stream_append_get_bytes(u3_noun); + u3_noun u3we_stream_append_read_bytes(u3_noun); + + u3_noun u3we_git_pak_expand_delta_object(u3_noun); + u3_noun u3we_git_protocol_stream_pkt_lines_on_band(u3_noun); + u3_noun u3we_bend_fun(u3_noun); u3_noun u3we_cold_fun(u3_noun); u3_noun u3we_cook_fun(u3_noun); From 7017245d8bb94ad26260262d450c7aa6eeadf62a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miko=C5=82aj=20Paraniak?= Date: Mon, 8 Jan 2024 20:33:45 +0800 Subject: [PATCH 05/12] Fix allocation across git jets --- BUILD.bazel | 11 + WORKSPACE.bazel | 19 ++ pkg/noun/jets/e/git.c | 394 +++++++++++++++++++++++---------- pkg/noun/jets/e/git_protocol.c | 42 ++-- pkg/noun/jets/e/stream.c | 21 +- pkg/noun/jets/e/zlib.c | 75 ++----- pkg/noun/jets/q.h | 2 +- pkg/noun/jets/tree.c | 15 +- 8 files changed, 356 insertions(+), 223 deletions(-) diff --git a/BUILD.bazel b/BUILD.bazel index 3569ed78a2..44aa66744b 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -212,3 +212,14 @@ alias( name = "urbit", actual = "//pkg/vere:urbit", ) + +load("@hedron_compile_commands//:refresh_compile_commands.bzl", "refresh_compile_commands") +refresh_compile_commands( + name = "refresh_compile_commands", + + # Specify the targets of interest. + # For example, specify a dict of targets and any flags required to build. + targets = { + "//:urbit": "", + }, +) diff --git a/WORKSPACE.bazel b/WORKSPACE.bazel index 39ddfe03e9..0f9579e5c5 100644 --- a/WORKSPACE.bazel +++ b/WORKSPACE.bazel @@ -419,3 +419,22 @@ container_pull( registry = "docker.io", repository = "alpine", ) + + +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") + + +# Hedron's Compile Commands Extractor for Bazel +# https://github.com/hedronvision/bazel-compile-commands-extractor +http_archive( + name = "hedron_compile_commands", + + # Replace the commit hash (daae6f40adfa5fdb7c89684cbe4d88b691c63b2d) in both places (below) with the latest (https://github.com/hedronvision/bazel-compile-commands-extractor/commits/main), rather than using the stale one here. + # Even better, set up Renovate and let it do the work for you (see "Suggestion: Updates" in the README). + url = "https://github.com/hedronvision/bazel-compile-commands-extractor/archive/eac41eefb5c19d9a2d2bcdd60d6989660288333d.tar.gz", + strip_prefix = "bazel-compile-commands-extractor-eac41eefb5c19d9a2d2bcdd60d6989660288333d", + # When you first run this tool, it'll recommend a sha256 hash to put here with a message like: "DEBUG: Rule 'hedron_compile_commands' indicated that a canonical reproducible form can be obtained by modifying arguments sha256 = ..." +) +load("@hedron_compile_commands//:workspace_setup.bzl", "hedron_compile_commands_setup") +hedron_compile_commands_setup() + diff --git a/pkg/noun/jets/e/git.c b/pkg/noun/jets/e/git.c index 2acb6b1ca3..b4fe1ac435 100644 --- a/pkg/noun/jets/e/git.c +++ b/pkg/noun/jets/e/git.c @@ -1,181 +1,339 @@ -#include "c3.h" +#include #include "jets/q.h" #include "jets/w.h" +#include "c3.h" #include "allocate.h" #include "noun.h" -#define FLUSH_PKT 0 -#define DELIM_PKT 1 -#define END_PKT 2 +#define OFS_DELTA (u3i_string("ofs-delta")) +#define REF_DELTA (u3i_string("ref-delta")) -#define PKT_LINE_MAX ((1 << 16) - 1 - 4) +static u3_atom _q_octs; -static c3_w _hex_dit(c3_y byt) -{ - // XX parse in the main function - // - u3_assert(('0' <= byt && byt <= '9') || ('a' <= byt && byt <= 'f')); +static c3_y* _unpack_octs(u3_atom p_octs, u3_atom* q_octs_p, c3_w* len_wp) { + + c3_y* buf_y; + c3_w len_w; + + u3_atom q_octs = *q_octs_p; + + u3_assert(c3y == u3a_is_cat(p_octs)); - c3_w dit_w; + if (c3y == u3a_is_cat(q_octs)) { - if (byt > '9') { - dit_w = (byt - 'a') + 10; + buf_y = (c3_y*)q_octs_p; + len_w = sizeof(q_octs); } else { - dit_w = (byt - '0'); + u3a_atom* a = u3a_to_ptr(q_octs); + buf_y = (c3_y*)a->buf_w; + len_w = u3r_met(3, q_octs); } - return dit_w; + u3_assert(len_w <= p_octs); + + *len_wp = len_w; + + return buf_y; } -u3_noun u3qe_git_protocol_stream_pkt_lines_on_band( - u3_atom band, - u3_noun sea){ +static inline int _read_octs_byte(c3_y* buf_y, c3_w* pos_wp, c3_w buf_len_w, c3_w len_w) { - if (c3n == u3a_is_cat(band)) { - u3m_bail(c3__fail); - } + c3_w pos_w = *pos_wp; + + u3_assert(buf_len_w <= len_w); + u3_assert(pos_w < len_w); - c3_w band_w; + c3_y bat_y = 0; - if (c3y != u3r_safe_word(band, &band_w)){ - u3m_bail(c3__fail); + if (pos_w < buf_len_w) { + bat_y = *(buf_y + pos_w); } - u3_atom pos, p_octs, q_octs; - u3x_mean(sea, 2, &pos, 6, &p_octs, 7, &q_octs, 0); + (*pos_wp)++; - c3_w pos_w, p_octs_w; + return bat_y; +} - if (c3n == u3r_safe_word(pos, &pos_w)) { - u3m_bail(c3__fail); - } +static c3_w _read_size(c3_y* buf_y, c3_w* pos_wp, c3_w buf_len_w, c3_w len_w) { - if (c3n == u3r_safe_word(p_octs, &p_octs_w)) { - u3m_bail(c3__fail); - } + c3_y bat_y = 0; + c3_w bits = 0; + c3_w size = 0; - if (c3n == u3a_is_atom(q_octs)) { - u3m_bail(c3__fail); - } + c3_w pos_w = *pos_wp; - c3_y *sea_y; + // fprintf(stderr, "_read_size: ***\r\n"); + while (pos_w < len_w) { + bat_y = _read_octs_byte(buf_y, &pos_w, buf_len_w, len_w); - if (c3y == u3a_is_cat(q_octs)) { - sea_y = (c3_y*)&q_octs; - } - else { - u3a_atom *a = u3a_to_ptr(q_octs); - // Verify octs sanity - // - u3_assert(p_octs_w == u3r_met(3, q_octs)); - sea_y = (c3_y*)a->buf_w; + // fprintf(stderr, "_read_size [%d/%d]: 0x%hhx\r\n", pos_w, len_w, bat_y); + + size += (bat_y & 0x7f) << bits; + bits += 7; + + if (!(bat_y & 0x80)) break; } - sea_y += pos_w; + *pos_wp = pos_w; + + return size; +} + +u3_noun u3qe_git_pak_expand_delta_object(u3_noun base, + u3_noun delta) { - c3_w len_w = PKT_LINE_MAX; - c3_w total_w = 0; - c3_y *buf_y = c3_malloc(len_w); + /* +$ raw-object [type=object-type data=stream:libstream] + */ + /* +$ pack-object $% raw-object + [%ofs-delta pos=@ud base-offset=@ud =octs] + [%ref-delta =octs] + == + +$ pack-delta-object $>(?(%ofs-delta %ref-delta) pack-object) + */ - if (buf_y == NULL) { - u3m_bail(c3__fail); + // base=raw-object + // + u3_atom base_type; + u3_noun base_data; + + u3_atom base_data_pos; + u3_atom base_data_p_octs; + u3_atom base_data_q_octs; + + u3x_cell(base, &base_type, &base_data); + u3x_trel(base_data, &base_data_pos, &base_data_p_octs, &base_data_q_octs); + + // delta=pack-object + // + u3_noun delta_type; + u3_noun delta_obj; + + u3_atom delta_pos; + u3_atom delta_base_offset; + u3_noun delta_octs; + + u3_atom delta_p_octs; + u3_atom delta_q_octs; + + u3x_cell(delta, &delta_type, &delta_obj); + + if ( c3y == u3r_sing(delta_type, OFS_DELTA) ) { + // fprintf(stderr, "Expanding %%ofs-delta object\r\n"); + u3x_trel(delta_obj, &delta_pos, &delta_base_offset, &delta_octs); + } + + if ( c3y == u3r_sing(delta_type, REF_DELTA) ) { + // fprintf(stderr, "Expanding %%ref-delta object\r\n"); + delta_octs = delta_obj; + u3_assert(false); } - c3_w pkt_len_w; - c3_y pkt_band_y; + u3x_cell(delta_octs, &delta_p_octs, &delta_q_octs); - while (pos_w < p_octs_w) { + c3_y* sea_y; + c3_y* sea_begin_y; + c3_w sea_pos_w; - /* - * Parse pkt-line length - */ - if (pos_w + 4 > p_octs_w) { - c3_free(buf_y); - fprintf(stderr, "u3qe_git__stream_pkt_lines_on_band: parsing failure\r\n"); - return u3_none; - } + c3_w sea_buf_len_w; + c3_w sea_len_w; - pkt_len_w = (_hex_dit(sea_y[0]) << 12) + - (_hex_dit(sea_y[1]) << 8) + - (_hex_dit(sea_y[2]) << 4) + - _hex_dit(sea_y[3]); - // fprintf(stderr, "u3qe_git__stream_pkt_lines_on_band: pos_w = %d, pkt_len_w = %d, sea_y = %.10s\r\n", pos_w, pkt_len_w, sea_y); + sea_y = _unpack_octs(delta_p_octs, &delta_q_octs, &sea_buf_len_w); + sea_begin_y = sea_y; + sea_len_w = delta_p_octs; + sea_pos_w = 0; - if (FLUSH_PKT == pkt_len_w) { - break; - } + c3_y* bas_y; + c3_y* bas_begin_y; - pos_w += 4; - sea_y += 4; - pkt_len_w -= 4; + c3_w bas_buf_len_w; + c3_w bas_len_w; - u3_assert(pkt_len_w <= PKT_LINE_MAX); + bas_y = _unpack_octs(base_data_p_octs, &base_data_q_octs, &bas_buf_len_w); + bas_begin_y = bas_y; + bas_len_w = base_data_p_octs; - /* - * Make space for pkt_len_w bytes in the buffer - */ - if (pos_w + pkt_len_w > p_octs_w) { - c3_free(buf_y); - u3m_bail(c3__fail); - } + /* Base object size (biz) and target + * object size (siz) + */ + // XX should exit cleanly + c3_w biz_w = _read_size(sea_y, &sea_pos_w, sea_buf_len_w, sea_len_w); + c3_w siz_w = _read_size(sea_y, &sea_pos_w, sea_buf_len_w, sea_len_w); - if (total_w + pkt_len_w > len_w) { + // fprintf(stderr, "u3qe__pak_expand_delta_object: biz_w = %d, siz_w = %d\r\n", biz_w, siz_w); - // Should we malloc & copy here? - // - c3_y *new_y = c3_malloc(len_w + 2*PKT_LINE_MAX); + // Base size mismatch + // + if (biz_w != (base_data_p_octs - base_data_pos)) { + fprintf(stderr, "bas_buf_len_w = %d, bas_len_w = %d\r\n", bas_buf_len_w, bas_len_w); + fprintf(stderr, "sea_pos = %d, sea_buf_len = %d, sea_len = %d\r\n", sea_pos_w, sea_buf_len_w, sea_len_w); + fprintf(stderr, "u3qe_git_pak_expand_delta_object: base (pos = %d) object size mismatch!\r\n", base_data_pos); + + // u3_assert(false); + // _free(); + return u3_none; + } - if ( new_y == NULL) { - c3_free(buf_y); - u3m_bail(c3__fail); - } + // Target buffer + u3i_slab sab_u; + u3i_slab_init(&sab_u, 3, siz_w); - memcpy(new_y, buf_y, len_w); - len_w += 2*PKT_LINE_MAX; + c3_y* tar_y = sab_u.buf_y; + c3_w tar_len_w = siz_w; - c3_free(buf_y); - buf_y = new_y; - } + c3_y bat_y; - /* - * Parse band - */ - pkt_band_y = *sea_y; + while (sea_pos_w < sea_len_w) { - sea_y += 1; - pos_w += 1; - pkt_len_w -= 1; + bat_y = _read_octs_byte(sea_y, &sea_pos_w, sea_buf_len_w, sea_len_w); - memmove(buf_y + total_w, sea_y, pkt_len_w); + // XX ?> (lth pos.sea p.octs.sea) + if ( 0x0 == bat_y ) { + fprintf(stderr, "u3qe__pak_expand_delta_object: hit reserved instruction 0x0\r\n"); - if (pkt_band_y == band_w) { - // fprintf(stderr, "u3we_git__stream_pkt_lines_on_band: accumulating %d bytes\r\n", pkt_len_w); - total_w += pkt_len_w; + u3m_bail(c3__fail); } else { - // buf_y[total_w + pkt_len_w] = 0; - // fprintf(stderr, "[band %d] %s\r\n", pkt_band_y, buf_y + total_w); + /* ADD instruction + */ + if (!(bat_y & 0x80)) { + + c3_w siz_w = bat_y & 0x7f; + + if (sea_len_w - sea_pos_w < siz_w) { + fprintf(stderr, "u3qe__pak_expand_delta_object: invalid add instruction\r\n"); + + u3_assert(false); + return u3_none; + } + + // fprintf(stderr, "u3qe__pak_expand_delta: ADD[siz_w = %d]\r\n", siz_w); + + if (tar_len_w < siz_w) { + fprintf(stderr, "u3qe__pak_expand_delta: ADD overflowed\r\n"); + + u3_assert(false); + return u3_none; + } + + // Some part to be copied falls inside + // the atom buffer + // + if (sea_buf_len_w > sea_pos_w) { + + c3_w cin_w = (sea_buf_len_w - sea_pos_w); + + if (siz_w < cin_w) { + memcpy(tar_y, sea_y + sea_pos_w, siz_w); + } + else { + memcpy(tar_y, sea_y + sea_pos_w, cin_w); + memset(tar_y+cin_w, 0, siz_w - cin_w); + } + } + else { + memset(tar_y, 0, siz_w); + } + + sea_pos_w += siz_w; + + tar_y += siz_w; + tar_len_w -= siz_w; + } + /* COPY instruction + */ + else { + + /* Retrieve offset and size + */ + c3_w off_w = 0; + c3_w siz_w = 0; + +#define _parse_cp_param(bit, var, shift) \ + { \ + if (bat_y & (bit)) { \ + if (!(sea_pos_w < sea_len_w)) { \ + u3_assert(false); \ + return u3_none; \ + } \ + var |= _read_octs_byte(sea_y, &sea_pos_w, sea_buf_len_w, sea_len_w) << shift; \ + } \ + } \ + + /* Parse offset + */ + _parse_cp_param(0x1, off_w, 0); + _parse_cp_param(0x2, off_w, 8); + _parse_cp_param(0x4, off_w, 16); + _parse_cp_param(0x8, off_w, 24); + + /* Parse size + */ + _parse_cp_param(0x10, siz_w, 0); + _parse_cp_param(0x20, siz_w, 8); + _parse_cp_param(0x40, siz_w, 16); + + if (siz_w == 0) { + siz_w = 0x10000; + } + + if (tar_len_w < siz_w || (bas_len_w - off_w) < siz_w) { + fprintf(stderr, "u3qe__pak_expand_delta: copy out of range\r\n"); + u3_assert(false); + return u3_none; + } + + // fprintf(stderr, "u3qe__pak_expand_delta: COPY[siz_w = %d, off_w = %d]\r\n", siz_w, off_w); + + // Region to be copied overlaps with the atom buffer + // + if (bas_buf_len_w > off_w) { + + c3_w cin_w = (bas_buf_len_w - off_w); + + // Region to be copied is wholly inside + // + if (siz_w < cin_w) { + memcpy(tar_y, bas_y + off_w, siz_w); + } + // Region to be copied is partially inside + // + else { + memcpy(tar_y, bas_y + off_w, cin_w); + memset(tar_y+cin_w, 0, siz_w - cin_w); + } + } + else { + memset(tar_y, 0, siz_w); + } + + tar_y += siz_w; + tar_len_w -= siz_w; + } } + } - sea_y += pkt_len_w; - pos_w += pkt_len_w; + if (tar_len_w) { + fprintf(stderr, "u3qe__pak_expand_delta: target object underfilled (%d bytes left)\r\n", tar_len_w); + u3_assert(false); + return u3_none; } - u3_noun octs_red = u3nc(u3i_word(total_w), u3i_bytes(total_w, buf_y)); - c3_free(buf_y); + u3_noun data = u3nc(0, u3nc(u3i_chub(siz_w), u3i_slab_mint(&sab_u))); + u3_noun rob = u3nc(u3k(base_type), data); - return u3nc(u3nc(u3i_word(0), octs_red), u3k(sea)); + return rob; } + u3_noun u3we_git_pak_expand_delta_object(u3_noun cor) { - u3_atom band; - u3_noun sea; + u3_noun base; + u3_noun delta; - u3x_mean(cor, u3x_sam_2, &band, - u3x_sam_3, &sea, 0); + u3x_mean(cor, u3x_sam_2, &base, + u3x_sam_3, &delta, 0); - return u3qe_git_protocol_stream_pkt_lines_on_band(band, sea); + return u3qe_git_pak_expand_delta_object(base, delta); } diff --git a/pkg/noun/jets/e/git_protocol.c b/pkg/noun/jets/e/git_protocol.c index 3b0fe5c1da..0f9e078097 100644 --- a/pkg/noun/jets/e/git_protocol.c +++ b/pkg/noun/jets/e/git_protocol.c @@ -79,11 +79,11 @@ u3_noun u3qe_git_protocol_stream_pkt_lines_on_band( c3_w len_w = PKT_LINE_MAX; c3_w total_w = 0; - c3_y *buf_y = c3_malloc(len_w); - if (buf_y == NULL) { - u3m_bail(c3__fail); - } + u3i_slab sab_u; + u3i_slab_init(&sab_u, 3, len_w); + + c3_y* buf_y = sab_u.buf_y; c3_w pkt_len_w; c3_y pkt_band_y; @@ -119,28 +119,10 @@ u3_noun u3qe_git_protocol_stream_pkt_lines_on_band( * Make space for pkt_len_w bytes in the buffer */ if (pos_w + pkt_len_w > p_octs_w) { - c3_free(buf_y); + u3i_slab_free(&sab_u); u3m_bail(c3__fail); } - if (total_w + pkt_len_w > len_w) { - - // Should we malloc & copy here? - // - c3_y *new_y = c3_malloc(len_w + 2*PKT_LINE_MAX); - - if ( new_y == NULL) { - c3_free(buf_y); - u3m_bail(c3__fail); - } - - memcpy(new_y, buf_y, len_w); - len_w += 2*PKT_LINE_MAX; - - c3_free(buf_y); - buf_y = new_y; - } - /* * Parse band */ @@ -150,10 +132,16 @@ u3_noun u3qe_git_protocol_stream_pkt_lines_on_band( pos_w += 1; pkt_len_w -= 1; - memmove(buf_y + total_w, sea_y, pkt_len_w); if (pkt_band_y == band_w) { - // fprintf(stderr, "u3we_git__stream_pkt_lines_on_band: accumulating %d bytes\r\n", pkt_len_w); + if (total_w + pkt_len_w > len_w) { + + len_w += 4*PKT_LINE_MAX; + u3i_slab_grow(&sab_u, 3, len_w); + buf_y = sab_u.buf_y; + } + + memcpy(buf_y + total_w, sea_y, pkt_len_w); total_w += pkt_len_w; } else { @@ -164,9 +152,7 @@ u3_noun u3qe_git_protocol_stream_pkt_lines_on_band( sea_y += pkt_len_w; pos_w += pkt_len_w; } - - u3_noun octs_red = u3nc(u3i_word(total_w), u3i_bytes(total_w, buf_y)); - c3_free(buf_y); + u3_noun octs_red = u3nc(u3i_word(total_w), u3i_slab_mint(&sab_u)); return u3nc(u3nc(u3i_word(0), octs_red), u3k(sea)); } diff --git a/pkg/noun/jets/e/stream.c b/pkg/noun/jets/e/stream.c index f3cc9311ba..83dcc65c65 100644 --- a/pkg/noun/jets/e/stream.c +++ b/pkg/noun/jets/e/stream.c @@ -86,8 +86,10 @@ u3_noun u3qe_stream_append_get_bytes(u3_atom n, u3_noun red, u3_noun sea) c3_i red_len_i = u3r_met(3, q_octs_red); c3_i new_red_len_i = p_octs_red_w + n_w; - c3_y *buf_y = c3_malloc(1+new_red_len_i); - buf_y[new_red_len_i] = 0; + u3i_slab sab_u; + u3i_slab_init(&sab_u, 3, new_red_len_i); + + c3_y* buf_y = sab_u.buf_y; // Copy over existing content in red // @@ -103,9 +105,7 @@ u3_noun u3qe_stream_append_get_bytes(u3_atom n, u3_noun red, u3_noun sea) u3_atom new_red = u3nt(u3k(pos_red), u3i_chub(new_red_len_i), - u3i_bytes(new_red_len_i, buf_y)); - - c3_free(buf_y); + u3i_slab_mint(&sab_u)); return u3nc(new_red, u3k(sea)); } @@ -203,8 +203,11 @@ u3_noun u3qe_stream_append_read_bytes(u3_atom n, u3_noun red, u3_noun sea) c3_i red_len_i = u3r_met(3, q_octs_red); c3_i new_red_len_i = p_octs_red_w + n_w; - c3_y *buf_y = c3_malloc(1+new_red_len_i); - buf_y[new_red_len_i] = 0; + u3i_slab sab_u; + u3i_slab_init(&sab_u, 3, new_red_len_i); + sab_u.buf_w[sab_u.len_w - 1] = 0; + + c3_y* buf_y = sab_u.buf_y; // Copy over existing content in red // @@ -220,9 +223,7 @@ u3_noun u3qe_stream_append_read_bytes(u3_atom n, u3_noun red, u3_noun sea) u3_atom new_red = u3nt(u3k(pos_red), u3i_chub(new_red_len_i), - u3i_bytes(new_red_len_i, buf_y)); - - c3_free(buf_y); + u3i_slab_mint(&sab_u)); return u3nc(new_red, u3nc(u3i_word(pos_sea_w + n_w), u3k(octs_sea))); } diff --git a/pkg/noun/jets/e/zlib.c b/pkg/noun/jets/e/zlib.c index da526655a0..e6b76197a9 100644 --- a/pkg/noun/jets/e/zlib.c +++ b/pkg/noun/jets/e/zlib.c @@ -22,7 +22,7 @@ u3_noun u3qe_zlib_expand(u3_atom pos, u3_noun byts) { u3_atom wid; u3_atom dat; - u3x_mean(byts, 2, &wid, 3, &dat, 0); + u3x_cell(byts, &wid, &dat); size_t sad_i = u3r_met(3, dat); @@ -64,7 +64,9 @@ u3_noun u3qe_zlib_expand(u3_atom pos, u3_noun byts) { // Size of the output buffer // size_t sob_i = OUTBUF_SZ; - c3_y* cuf_y = c3_malloc(sob_i); + u3i_slab sab_u; + u3i_slab_init(&sab_u, 3, sob_i); + c3_y* cuf_y = sab_u.buf_y; u3_assert(cuf_y != NULL); @@ -76,7 +78,7 @@ u3_noun u3qe_zlib_expand(u3_atom pos, u3_noun byts) { if( Z_OK != zas_w) { fprintf(stderr, "u3qe_zlib_expand: error while initializing Zlib, zas_w = %d\r\n", zas_w); - c3_free(cuf_y); + u3i_slab_free(&sab_u); return u3_none; } @@ -88,70 +90,17 @@ u3_noun u3qe_zlib_expand(u3_atom pos, u3_noun byts) { } if (zea.avail_out == 0) { - - c3_y* new_y = c3_realloc(cuf_y, sob_i + OUTBUF_SZ); - u3_assert(new_y != NULL); - - cuf_y = new_y; + u3i_slab_grow(&sab_u, 3, sob_i + OUTBUF_SZ); + cuf_y = sab_u.buf_y + sob_i; zea.avail_out = OUTBUF_SZ; - + zea.next_out = cuf_y; u3_assert(sob_i == zea.total_out); - - zea.next_out = cuf_y + sob_i; - sob_i += OUTBUF_SZ; } } - if (zas_w != Z_STREAM_END) { - - fprintf(stderr, "u3qe_zlib_expand: %d input bytes\r\n", sad_i); - fprintf(stderr, "u3qe_zlib_expand: processed %d bytes\r\n", zea.total_in); - fprintf(stderr, "u3qe_zlib_expand: uncompressed %d bytes\r\n", zea.total_out); - fprintf(stderr, "u3qe_zlib_expand: error while expanding, zas_w = %d, msg = %s\r\n", zas_w, zea.msg); - - // Dump ZLIB stream - // - time_t now; - time(&now); - char filename[256]; - if ( zas_w != Z_STREAM_END ) { - sprintf(filename, "error-stream-%d.dat", now); - } - else { - sprintf(filename, "good-stream-%d.dat", now); - } - - fprintf(stderr, "u3qe_zlib_expand: dumping corrupted stream to %s\r\n", filename); - FILE* file= fopen(filename, "w"); - assert(file != NULL); - - // byt_y + pos_d, for zea.total_in bytes - // - fwrite(byt_y+pos_d, sizeof(char), zea.total_in, file); - - fclose(file); - - if ( zas_w != Z_STREAM_END ) { - c3_free(cuf_y); - return u3_none; - } - - } - size_t len_i = sob_i - zea.avail_out; - c3_y *buf_y = c3_malloc(len_i); - - if (buf_y == NULL) { - c3_free(cuf_y); - u3m_bail(c3__meme); - } - - memcpy(buf_y, cuf_y, len_i); - - u3_atom len_a = u3i_chub(len_i); - pos_d += zea.total_in; zas_w = inflateEnd(&zea); @@ -160,12 +109,14 @@ u3_noun u3qe_zlib_expand(u3_atom pos, u3_noun byts) { fprintf(stderr, "u3qe_zlib_expand: zlib stream inconsistent upon finish, zas_w = %d\r\n", zas_w); - c3_free(cuf_y); + u3i_slab_free(&sab_u); return u3_none; } - c3_free(cuf_y); - return u3nc(u3nc(len_a, u3i_bytes(len_i, buf_y)), + u3_atom len_a = u3i_chub(len_i); + u3_atom buf_a = u3i_slab_mint(&sab_u); + + return u3nc(u3nc(len_a, buf_a), u3nc(u3i_chub(pos_d), u3k(byts))); } diff --git a/pkg/noun/jets/q.h b/pkg/noun/jets/q.h index 05090b8bd1..31ba9027e8 100644 --- a/pkg/noun/jets/q.h +++ b/pkg/noun/jets/q.h @@ -164,8 +164,8 @@ u3_noun u3qe_stream_append_get_bytes(u3_atom n, u3_noun red, u3_noun sea); u3_noun u3qe_stream_append_read_bytes(u3_atom n, u3_noun red, u3_noun sea); - u3_noun u3qe_git_pak_expand_delta_object(u3_noun, u3_noun); u3_noun u3qe_git_protocol_stream_pkt_lines_on_band(u3_atom band, u3_noun sea); + u3_noun u3qe_git_pak_expand_delta_object(u3_noun, u3_noun); u3_noun u3qeo_raw(u3_atom, u3_atom); diff --git a/pkg/noun/jets/tree.c b/pkg/noun/jets/tree.c index 319b6a6e61..3c2fdabf71 100644 --- a/pkg/noun/jets/tree.c +++ b/pkg/noun/jets/tree.c @@ -2158,10 +2158,17 @@ static u3j_core _139_hex_git_protocol_d[] = {} }; -/* - * Git core - */ +static u3j_harm _139_hex_git_pak_expand_delta_object_a[] = + {{".2", u3we_git_pak_expand_delta_object, c3y}, {}}; + +static u3j_core _139_hex_git_pak_d[] = + {{"expand-delta-object", 7, _139_hex_git_pak_expand_delta_object_a, c3y}, {}}; +static u3j_core _139_hex_git_d[] = + { + {"pak", 3, 0, _139_hex_git_pak_d, no_hashes}, + {} + }; static u3j_core _139_hex_d[] = { { "lore", 63, _140_hex_lore_a, 0, no_hashes }, { "leer", 63, _140_hex_leer_a, 0, no_hashes }, @@ -2182,7 +2189,7 @@ static u3j_core _139_hex_d[] = { "json", 31, 0, _139_hex_json_d, no_hashes }, { "zlib", 31, 0, _139_hex_zlib_d, no_hashes}, { "stream", 31, 0, _139_hex_stream_d, no_hashes}, - { "git", 31, 0, _139_hex_git_d, no_hashes}, + { "git" , 31, 0, _139_hex_git_d, no_hashes}, // XX this be under git, even though the cores are separate { "git-protocol", 31, 0, _139_hex_git_protocol_d, no_hashes}, {} From 59dfe7187de6a5375cca4b1012736fc4b9fb0539 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miko=C5=82aj=20Paraniak?= Date: Tue, 9 Jan 2024 18:27:43 +0800 Subject: [PATCH 06/12] Adjust git jets --- pkg/noun/jets/e/{git_protocol.c => git_io.c} | 6 +++--- pkg/noun/jets/q.h | 2 +- pkg/noun/jets/tree.c | 11 +++++------ pkg/noun/jets/w.h | 2 +- 4 files changed, 10 insertions(+), 11 deletions(-) rename pkg/noun/jets/e/{git_protocol.c => git_io.c} (94%) diff --git a/pkg/noun/jets/e/git_protocol.c b/pkg/noun/jets/e/git_io.c similarity index 94% rename from pkg/noun/jets/e/git_protocol.c rename to pkg/noun/jets/e/git_io.c index 0f9e078097..bb031cf037 100644 --- a/pkg/noun/jets/e/git_protocol.c +++ b/pkg/noun/jets/e/git_io.c @@ -31,7 +31,7 @@ static c3_w _hex_dit(c3_y byt) return dit_w; } -u3_noun u3qe_git_protocol_stream_pkt_lines_on_band( +u3_noun u3qe_git_io_stream_pkt_lines_on_band( u3_atom band, u3_noun sea){ @@ -156,7 +156,7 @@ u3_noun u3qe_git_protocol_stream_pkt_lines_on_band( return u3nc(u3nc(u3i_word(0), octs_red), u3k(sea)); } -u3_noun u3we_git_protocol_stream_pkt_lines_on_band(u3_noun cor) { +u3_noun u3we_git_io_stream_pkt_lines_on_band(u3_noun cor) { u3_atom band; u3_noun sea; @@ -164,5 +164,5 @@ u3_noun u3we_git_protocol_stream_pkt_lines_on_band(u3_noun cor) { u3x_mean(cor, u3x_sam_2, &band, u3x_sam_3, &sea, 0); - return u3qe_git_protocol_stream_pkt_lines_on_band(band, sea); + return u3qe_git_io_stream_pkt_lines_on_band(band, sea); } diff --git a/pkg/noun/jets/q.h b/pkg/noun/jets/q.h index 31ba9027e8..1c8886a242 100644 --- a/pkg/noun/jets/q.h +++ b/pkg/noun/jets/q.h @@ -164,7 +164,7 @@ u3_noun u3qe_stream_append_get_bytes(u3_atom n, u3_noun red, u3_noun sea); u3_noun u3qe_stream_append_read_bytes(u3_atom n, u3_noun red, u3_noun sea); - u3_noun u3qe_git_protocol_stream_pkt_lines_on_band(u3_atom band, u3_noun sea); + u3_noun u3qe_git_io_stream_pkt_lines_on_band(u3_atom band, u3_noun sea); u3_noun u3qe_git_pak_expand_delta_object(u3_noun, u3_noun); u3_noun u3qeo_raw(u3_atom, u3_atom); diff --git a/pkg/noun/jets/tree.c b/pkg/noun/jets/tree.c index 3c2fdabf71..62ebef679c 100644 --- a/pkg/noun/jets/tree.c +++ b/pkg/noun/jets/tree.c @@ -2149,12 +2149,12 @@ static u3j_core _139_hex_stream_d[] = /* * Git protocol */ -static u3j_harm _139_hex_git_protocol_stream_pkt_lines_on_band_a[] = - {{".2", u3we_git_protocol_stream_pkt_lines_on_band, c3y}, {}}; +static u3j_harm _139_hex_git_io_stream_pkt_lines_on_band_a[] = + {{".2", u3we_git_io_stream_pkt_lines_on_band, c3y}, {}}; -static u3j_core _139_hex_git_protocol_d[] = +static u3j_core _139_hex_git_io_d[] = { - { "stream-pkt-lines-on-band", 7, _139_hex_git_protocol_stream_pkt_lines_on_band_a, 0, no_hashes}, + { "stream-pkt-lines-on-band", 7, _139_hex_git_io_stream_pkt_lines_on_band_a, 0, no_hashes}, {} }; @@ -2190,8 +2190,7 @@ static u3j_core _139_hex_d[] = { "zlib", 31, 0, _139_hex_zlib_d, no_hashes}, { "stream", 31, 0, _139_hex_stream_d, no_hashes}, { "git" , 31, 0, _139_hex_git_d, no_hashes}, - // XX this be under git, even though the cores are separate - { "git-protocol", 31, 0, _139_hex_git_protocol_d, no_hashes}, + { "git-io", 31, 0, _139_hex_git_io_d, no_hashes}, {} }; diff --git a/pkg/noun/jets/w.h b/pkg/noun/jets/w.h index d95e847f50..3a72856b71 100644 --- a/pkg/noun/jets/w.h +++ b/pkg/noun/jets/w.h @@ -231,8 +231,8 @@ u3_noun u3we_stream_append_get_bytes(u3_noun); u3_noun u3we_stream_append_read_bytes(u3_noun); + u3_noun u3we_git_io_stream_pkt_lines_on_band(u3_noun); u3_noun u3we_git_pak_expand_delta_object(u3_noun); - u3_noun u3we_git_protocol_stream_pkt_lines_on_band(u3_noun); u3_noun u3we_bend_fun(u3_noun); u3_noun u3we_cold_fun(u3_noun); From 508b874bd9118d686b3ee0829bbe40d22efcf4fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miko=C5=82aj=20Paraniak?= Date: Fri, 12 Jan 2024 07:31:08 +0800 Subject: [PATCH 07/12] jets: adjust git pak jet, +pak is now a door --- pkg/noun/jets/tree.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/noun/jets/tree.c b/pkg/noun/jets/tree.c index 62ebef679c..4940137d72 100644 --- a/pkg/noun/jets/tree.c +++ b/pkg/noun/jets/tree.c @@ -2166,7 +2166,7 @@ static u3j_core _139_hex_git_pak_d[] = static u3j_core _139_hex_git_d[] = { - {"pak", 3, 0, _139_hex_git_pak_d, no_hashes}, + {"pak", 7, 0, _139_hex_git_pak_d, no_hashes}, {} }; static u3j_core _139_hex_d[] = From 0f30344386fadc053ba4ca011bbad5fbf4112c2b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miko=C5=82aj=20Paraniak?= Date: Wed, 20 Mar 2024 16:58:00 +0800 Subject: [PATCH 08/12] Refactor code; Jet write-pkt-lines-on-band:git-http --- pkg/noun/jets/e/git.c | 30 ++-- pkg/noun/jets/e/git_http.c | 287 +++++++++++++++++++++++++++++++++++++ pkg/noun/jets/e/git_io.c | 168 ---------------------- pkg/noun/jets/e/zlib.c | 7 +- pkg/noun/jets/q.h | 6 +- pkg/noun/jets/tree.c | 30 ++-- pkg/noun/jets/w.h | 6 +- 7 files changed, 326 insertions(+), 208 deletions(-) create mode 100644 pkg/noun/jets/e/git_http.c delete mode 100644 pkg/noun/jets/e/git_io.c diff --git a/pkg/noun/jets/e/git.c b/pkg/noun/jets/e/git.c index b4fe1ac435..8c7d7d0856 100644 --- a/pkg/noun/jets/e/git.c +++ b/pkg/noun/jets/e/git.c @@ -82,7 +82,7 @@ static c3_w _read_size(c3_y* buf_y, c3_w* pos_wp, c3_w buf_len_w, c3_w len_w) { return size; } -u3_noun u3qe_git_pak_expand_delta_object(u3_noun base, +u3_noun u3qe_git_pack_expand_delta_object(u3_noun base, u3_noun delta) { /* +$ raw-object [type=object-type data=stream:libstream] @@ -97,13 +97,14 @@ u3_noun u3qe_git_pak_expand_delta_object(u3_noun base, // base=raw-object // u3_atom base_type; + u3_atom base_size; u3_noun base_data; u3_atom base_data_pos; u3_atom base_data_p_octs; u3_atom base_data_q_octs; - u3x_cell(base, &base_type, &base_data); + u3x_trel(base, &base_type, &base_size, &base_data); u3x_trel(base_data, &base_data_pos, &base_data_p_octs, &base_data_q_octs); // delta=pack-object @@ -152,6 +153,7 @@ u3_noun u3qe_git_pak_expand_delta_object(u3_noun base, c3_w bas_len_w; bas_y = _unpack_octs(base_data_p_octs, &base_data_q_octs, &bas_buf_len_w); + // u3_assert(base_size == (base_data_p_octs - base_data_pos)); bas_begin_y = bas_y; bas_len_w = base_data_p_octs; @@ -162,14 +164,14 @@ u3_noun u3qe_git_pak_expand_delta_object(u3_noun base, c3_w biz_w = _read_size(sea_y, &sea_pos_w, sea_buf_len_w, sea_len_w); c3_w siz_w = _read_size(sea_y, &sea_pos_w, sea_buf_len_w, sea_len_w); - // fprintf(stderr, "u3qe__pak_expand_delta_object: biz_w = %d, siz_w = %d\r\n", biz_w, siz_w); + // fprintf(stderr, "u3qe__pack_expand_delta_object: biz_w = %d, siz_w = %d\r\n", biz_w, siz_w); // Base size mismatch // if (biz_w != (base_data_p_octs - base_data_pos)) { fprintf(stderr, "bas_buf_len_w = %d, bas_len_w = %d\r\n", bas_buf_len_w, bas_len_w); fprintf(stderr, "sea_pos = %d, sea_buf_len = %d, sea_len = %d\r\n", sea_pos_w, sea_buf_len_w, sea_len_w); - fprintf(stderr, "u3qe_git_pak_expand_delta_object: base (pos = %d) object size mismatch!\r\n", base_data_pos); + fprintf(stderr, "u3qe_git_pack_expand_delta_object: base (pos = %d) object size mismatch!\r\n", base_data_pos); // u3_assert(false); // _free(); @@ -191,7 +193,7 @@ u3_noun u3qe_git_pak_expand_delta_object(u3_noun base, // XX ?> (lth pos.sea p.octs.sea) if ( 0x0 == bat_y ) { - fprintf(stderr, "u3qe__pak_expand_delta_object: hit reserved instruction 0x0\r\n"); + fprintf(stderr, "u3qe__pack_expand_delta_object: hit reserved instruction 0x0\r\n"); u3m_bail(c3__fail); } @@ -203,16 +205,16 @@ u3_noun u3qe_git_pak_expand_delta_object(u3_noun base, c3_w siz_w = bat_y & 0x7f; if (sea_len_w - sea_pos_w < siz_w) { - fprintf(stderr, "u3qe__pak_expand_delta_object: invalid add instruction\r\n"); + fprintf(stderr, "u3qe__pack_expand_delta_object: invalid add instruction\r\n"); u3_assert(false); return u3_none; } - // fprintf(stderr, "u3qe__pak_expand_delta: ADD[siz_w = %d]\r\n", siz_w); + // fprintf(stderr, "u3qe__pack_expand_delta: ADD[siz_w = %d]\r\n", siz_w); if (tar_len_w < siz_w) { - fprintf(stderr, "u3qe__pak_expand_delta: ADD overflowed\r\n"); + fprintf(stderr, "u3qe__pack_expand_delta: ADD overflowed\r\n"); u3_assert(false); return u3_none; @@ -280,12 +282,12 @@ u3_noun u3qe_git_pak_expand_delta_object(u3_noun base, } if (tar_len_w < siz_w || (bas_len_w - off_w) < siz_w) { - fprintf(stderr, "u3qe__pak_expand_delta: copy out of range\r\n"); + fprintf(stderr, "u3qe__pack_expand_delta: copy out of range\r\n"); u3_assert(false); return u3_none; } - // fprintf(stderr, "u3qe__pak_expand_delta: COPY[siz_w = %d, off_w = %d]\r\n", siz_w, off_w); + // fprintf(stderr, "u3qe__pack_expand_delta: COPY[siz_w = %d, off_w = %d]\r\n", siz_w, off_w); // Region to be copied overlaps with the atom buffer // @@ -316,18 +318,18 @@ u3_noun u3qe_git_pak_expand_delta_object(u3_noun base, } if (tar_len_w) { - fprintf(stderr, "u3qe__pak_expand_delta: target object underfilled (%d bytes left)\r\n", tar_len_w); + fprintf(stderr, "u3qe__pack_expand_delta: target object underfilled (%d bytes left)\r\n", tar_len_w); u3_assert(false); return u3_none; } u3_noun data = u3nc(0, u3nc(u3i_chub(siz_w), u3i_slab_mint(&sab_u))); - u3_noun rob = u3nc(u3k(base_type), data); + u3_noun rob = u3nt(u3k(base_type), u3k(base_size), data); return rob; } -u3_noun u3we_git_pak_expand_delta_object(u3_noun cor) { +u3_noun u3we_git_pack_expand_delta_object(u3_noun cor) { u3_noun base; u3_noun delta; @@ -335,5 +337,5 @@ u3_noun u3we_git_pak_expand_delta_object(u3_noun cor) { u3x_mean(cor, u3x_sam_2, &base, u3x_sam_3, &delta, 0); - return u3qe_git_pak_expand_delta_object(base, delta); + return u3qe_git_pack_expand_delta_object(base, delta); } diff --git a/pkg/noun/jets/e/git_http.c b/pkg/noun/jets/e/git_http.c new file mode 100644 index 0000000000..523bcbbbc7 --- /dev/null +++ b/pkg/noun/jets/e/git_http.c @@ -0,0 +1,287 @@ + +#include "c3.h" + +#include "jets/q.h" +#include "jets/w.h" + +#include "allocate.h" +#include "noun.h" + +#define FLUSH_PKT 0 +#define DELIM_PKT 1 +#define END_PKT 2 + +#define PKT_LINE_MAX ((1 << 16) - 1 - 4) + +static c3_w _hex_dit(c3_y byt) +{ + // XX parse in the main function + // + u3_assert(('0' <= byt && byt <= '9') || ('a' <= byt && byt <= 'f')); + + c3_w dit_w; + + if (byt > '9') { + dit_w = (byt - 'a') + 10; + } + else { + dit_w = (byt - '0'); + } + + return dit_w; +} +static c3_y _to_hex_dit(c3_w byt) +{ + // XX parse in the main function + // + u3_assert(byt < 0xff); + + if (byt < 10) { + return byt + '0'; + } + else { + return (byt - 10) + 'a'; + } +} + +u3_noun u3qe_git_http_read_pkt_lines_on_band( + u3_noun sea, + u3_atom band){ + + if (c3n == u3a_is_cat(band)) { + u3m_bail(c3__fail); + } + + c3_w band_w; + + if (c3y != u3r_safe_word(band, &band_w)){ + u3m_bail(c3__fail); + } + + u3_atom pos, p_octs, q_octs; + u3x_mean(sea, 2, &pos, 6, &p_octs, 7, &q_octs, 0); + + c3_w pos_w, p_octs_w; + + if (c3n == u3r_safe_word(pos, &pos_w)) { + u3m_bail(c3__fail); + } + + if (c3n == u3r_safe_word(p_octs, &p_octs_w)) { + u3m_bail(c3__fail); + } + + if (c3n == u3a_is_atom(q_octs)) { + u3m_bail(c3__fail); + } + + c3_y *sea_y; + + if (c3y == u3a_is_cat(q_octs)) { + sea_y = (c3_y*)&q_octs; + } + else { + u3a_atom *a = u3a_to_ptr(q_octs); + // Verify octs sanity + // + u3_assert(p_octs_w == u3r_met(3, q_octs)); + sea_y = (c3_y*)a->buf_w; + } + + sea_y += pos_w; + + c3_w len_w = PKT_LINE_MAX; + c3_w total_w = 0; + + u3i_slab sab_u; + u3i_slab_init(&sab_u, 3, len_w); + + c3_y* buf_y = sab_u.buf_y; + + c3_w pkt_len_w; + c3_y pkt_band_y; + + while (pos_w < p_octs_w) { + + /* + * Parse pkt-line length + */ + if (pos_w + 4 > p_octs_w) { + c3_free(buf_y); + fprintf(stderr, "u3qe_git__read_pkt_lines_on_band: parsing failure\r\n"); + return u3_none; + } + + pkt_len_w = (_hex_dit(sea_y[0]) << 12) + + (_hex_dit(sea_y[1]) << 8) + + (_hex_dit(sea_y[2]) << 4) + + _hex_dit(sea_y[3]); + // fprintf(stderr, "u3qe_git__read_pkt_lines_on_band: pos_w = %d, pkt_len_w = %d, sea_y = %.10s\r\n", pos_w, pkt_len_w, sea_y); + + if (FLUSH_PKT == pkt_len_w) { + break; + } + + pos_w += 4; + sea_y += 4; + pkt_len_w -= 4; + + u3_assert(pkt_len_w <= PKT_LINE_MAX); + + /* + * Make space for pkt_len_w bytes in the buffer + */ + if (pos_w + pkt_len_w > p_octs_w) { + u3i_slab_free(&sab_u); + u3m_bail(c3__fail); + } + + /* + * Parse band + */ + pkt_band_y = *sea_y; + + sea_y += 1; + pos_w += 1; + pkt_len_w -= 1; + + + if (pkt_band_y == band_w) { + if (total_w + pkt_len_w > len_w) { + + len_w += 4*PKT_LINE_MAX; + u3i_slab_grow(&sab_u, 3, len_w); + buf_y = sab_u.buf_y; + } + + memcpy(buf_y + total_w, sea_y, pkt_len_w); + total_w += pkt_len_w; + } + else { + // buf_y[total_w + pkt_len_w] = 0; + // fprintf(stderr, "[band %d] %s\r\n", pkt_band_y, buf_y + total_w); + } + + sea_y += pkt_len_w; + pos_w += pkt_len_w; + } + u3_noun octs_red = u3nc(u3i_word(total_w), u3i_slab_mint(&sab_u)); + + return u3nc(u3nc(u3i_word(0), octs_red), u3k(sea)); +} +u3_noun u3we_git_http_read_pkt_lines_on_band(u3_noun cor) { + + u3_noun sea; + u3_atom band; + + u3x_mean(cor, u3x_sam_2, &sea, + u3x_sam_3, &band, 0); + + return u3qe_git_http_read_pkt_lines_on_band(sea, band); +} + +#define CHUNK_SZ ((1 << 13) - 4 - 1) + +/* XX handle octs with p < met q and p > met q */ +u3_noun u3qe_git_http_write_pkt_lines_on_band( + u3_noun sea, + u3_atom band){ + + if (c3n == u3a_is_cat(band)) { + u3m_bail(c3__fail); + } + + c3_w band_w; + + if (c3y != u3r_safe_word(band, &band_w)){ + u3m_bail(c3__fail); + } + + u3_atom pos, p_octs, q_octs; + u3x_mean(sea, 2, &pos, 6, &p_octs, 7, &q_octs, 0); + + c3_w pos_w, p_octs_w; + + if (c3n == u3r_safe_word(pos, &pos_w)) { + u3m_bail(c3__fail); + } + + if (c3n == u3r_safe_word(p_octs, &p_octs_w)) { + u3m_bail(c3__fail); + } + + if (c3n == u3a_is_atom(q_octs)) { + u3m_bail(c3__fail); + } + + c3_y *sea_y; + + // XX This is broken if p does not match + // the length of q + // + if (c3y == u3a_is_cat(q_octs)) { + sea_y = (c3_y*)&q_octs; + } + + else { + u3a_atom *a = u3a_to_ptr(q_octs); + // Verify octs sanity + // + u3_assert(p_octs_w == u3r_met(3, q_octs)); + sea_y = (c3_y*)a->buf_w; + } + + sea_y += pos_w; + + // XX handle length properly + const c3_w len_w = (p_octs_w - pos_w); + const c3_w num_lines_w = (len_w / CHUNK_SZ) + ((len_w % CHUNK_SZ) ? 1 : 0); + // 4 bytes for packet length, 1 byte for band + // + const c3_w out_len_w = len_w + num_lines_w * 5; + + u3i_slab sab_u; + u3i_slab_init(&sab_u, 3, out_len_w); + + c3_y* buf_y = sab_u.buf_y; + + c3_w rem_w = len_w; + + while (rem_w) { + const c3_w copy_len_w = (rem_w < CHUNK_SZ) ? rem_w : CHUNK_SZ; + const c3_w line_len_w = copy_len_w + 5; + + // Write line legth + // + buf_y[0] = _to_hex_dit((line_len_w >> 12) & 0xf); + buf_y[1] = _to_hex_dit((line_len_w >> 8) & 0xf); + buf_y[2] = _to_hex_dit((line_len_w >> 4) & 0xf); + buf_y[3] = _to_hex_dit(line_len_w & 0xf); + + // Write band + // + buf_y[4] = 1; + + buf_y += 5; + + memcpy(buf_y, sea_y, copy_len_w); + + buf_y += copy_len_w; + sea_y += copy_len_w; + rem_w -= copy_len_w; + } + + u3_assert(buf_y == sab_u.buf_y + out_len_w); + + return u3nc(u3i_chub(out_len_w), u3i_slab_mint(&sab_u)); +} +u3_noun u3we_git_http_write_pkt_lines_on_band(u3_noun cor) { + + u3_noun sea; + u3_atom band; + + u3x_mean(cor, u3x_sam_2, &sea, + u3x_sam_3, &band, 0); + + return u3qe_git_http_write_pkt_lines_on_band(sea, band); +} diff --git a/pkg/noun/jets/e/git_io.c b/pkg/noun/jets/e/git_io.c deleted file mode 100644 index bb031cf037..0000000000 --- a/pkg/noun/jets/e/git_io.c +++ /dev/null @@ -1,168 +0,0 @@ - -#include "c3.h" - -#include "jets/q.h" -#include "jets/w.h" - -#include "allocate.h" -#include "noun.h" - -#define FLUSH_PKT 0 -#define DELIM_PKT 1 -#define END_PKT 2 - -#define PKT_LINE_MAX ((1 << 16) - 1 - 4) - -static c3_w _hex_dit(c3_y byt) -{ - // XX parse in the main function - // - u3_assert(('0' <= byt && byt <= '9') || ('a' <= byt && byt <= 'f')); - - c3_w dit_w; - - if (byt > '9') { - dit_w = (byt - 'a') + 10; - } - else { - dit_w = (byt - '0'); - } - - return dit_w; -} - -u3_noun u3qe_git_io_stream_pkt_lines_on_band( - u3_atom band, - u3_noun sea){ - - if (c3n == u3a_is_cat(band)) { - u3m_bail(c3__fail); - } - - c3_w band_w; - - if (c3y != u3r_safe_word(band, &band_w)){ - u3m_bail(c3__fail); - } - - u3_atom pos, p_octs, q_octs; - u3x_mean(sea, 2, &pos, 6, &p_octs, 7, &q_octs, 0); - - c3_w pos_w, p_octs_w; - - if (c3n == u3r_safe_word(pos, &pos_w)) { - u3m_bail(c3__fail); - } - - if (c3n == u3r_safe_word(p_octs, &p_octs_w)) { - u3m_bail(c3__fail); - } - - if (c3n == u3a_is_atom(q_octs)) { - u3m_bail(c3__fail); - } - - c3_y *sea_y; - - if (c3y == u3a_is_cat(q_octs)) { - sea_y = (c3_y*)&q_octs; - } - else { - u3a_atom *a = u3a_to_ptr(q_octs); - // Verify octs sanity - // - u3_assert(p_octs_w == u3r_met(3, q_octs)); - sea_y = (c3_y*)a->buf_w; - } - - sea_y += pos_w; - - c3_w len_w = PKT_LINE_MAX; - c3_w total_w = 0; - - u3i_slab sab_u; - u3i_slab_init(&sab_u, 3, len_w); - - c3_y* buf_y = sab_u.buf_y; - - c3_w pkt_len_w; - c3_y pkt_band_y; - - while (pos_w < p_octs_w) { - - /* - * Parse pkt-line length - */ - if (pos_w + 4 > p_octs_w) { - c3_free(buf_y); - fprintf(stderr, "u3qe_git__stream_pkt_lines_on_band: parsing failure\r\n"); - return u3_none; - } - - pkt_len_w = (_hex_dit(sea_y[0]) << 12) + - (_hex_dit(sea_y[1]) << 8) + - (_hex_dit(sea_y[2]) << 4) + - _hex_dit(sea_y[3]); - // fprintf(stderr, "u3qe_git__stream_pkt_lines_on_band: pos_w = %d, pkt_len_w = %d, sea_y = %.10s\r\n", pos_w, pkt_len_w, sea_y); - - if (FLUSH_PKT == pkt_len_w) { - break; - } - - pos_w += 4; - sea_y += 4; - pkt_len_w -= 4; - - u3_assert(pkt_len_w <= PKT_LINE_MAX); - - /* - * Make space for pkt_len_w bytes in the buffer - */ - if (pos_w + pkt_len_w > p_octs_w) { - u3i_slab_free(&sab_u); - u3m_bail(c3__fail); - } - - /* - * Parse band - */ - pkt_band_y = *sea_y; - - sea_y += 1; - pos_w += 1; - pkt_len_w -= 1; - - - if (pkt_band_y == band_w) { - if (total_w + pkt_len_w > len_w) { - - len_w += 4*PKT_LINE_MAX; - u3i_slab_grow(&sab_u, 3, len_w); - buf_y = sab_u.buf_y; - } - - memcpy(buf_y + total_w, sea_y, pkt_len_w); - total_w += pkt_len_w; - } - else { - // buf_y[total_w + pkt_len_w] = 0; - // fprintf(stderr, "[band %d] %s\r\n", pkt_band_y, buf_y + total_w); - } - - sea_y += pkt_len_w; - pos_w += pkt_len_w; - } - u3_noun octs_red = u3nc(u3i_word(total_w), u3i_slab_mint(&sab_u)); - - return u3nc(u3nc(u3i_word(0), octs_red), u3k(sea)); -} -u3_noun u3we_git_io_stream_pkt_lines_on_band(u3_noun cor) { - - u3_atom band; - u3_noun sea; - - u3x_mean(cor, u3x_sam_2, &band, - u3x_sam_3, &sea, 0); - - return u3qe_git_io_stream_pkt_lines_on_band(band, sea); -} diff --git a/pkg/noun/jets/e/zlib.c b/pkg/noun/jets/e/zlib.c index e6b76197a9..9d3489de13 100644 --- a/pkg/noun/jets/e/zlib.c +++ b/pkg/noun/jets/e/zlib.c @@ -43,11 +43,6 @@ u3_noun u3qe_zlib_expand(u3_atom pos, u3_noun byts) { byt_y = (c3_y*)vat_u->buf_w; } - /* Leading zeros do not make sense - * for a Zlib stream - */ - u3_assert(wid_d == sad_i); - z_stream zea; c3_w zas_w; @@ -73,7 +68,7 @@ u3_noun u3qe_zlib_expand(u3_atom pos, u3_noun byts) { zea.next_out = cuf_y; zea.avail_out = sob_i; - zas_w = inflateInit(&zea); + zas_w = inflateInit2(&zea, 47); if( Z_OK != zas_w) { fprintf(stderr, "u3qe_zlib_expand: error while initializing Zlib, zas_w = %d\r\n", zas_w); diff --git a/pkg/noun/jets/q.h b/pkg/noun/jets/q.h index 1c8886a242..9dc47b054a 100644 --- a/pkg/noun/jets/q.h +++ b/pkg/noun/jets/q.h @@ -164,8 +164,10 @@ u3_noun u3qe_stream_append_get_bytes(u3_atom n, u3_noun red, u3_noun sea); u3_noun u3qe_stream_append_read_bytes(u3_atom n, u3_noun red, u3_noun sea); - u3_noun u3qe_git_io_stream_pkt_lines_on_band(u3_atom band, u3_noun sea); - u3_noun u3qe_git_pak_expand_delta_object(u3_noun, u3_noun); + u3_noun u3qe_git_http_read_pkt_lines_on_band(u3_atom band, u3_noun sea); + u3_noun u3qe_git_http_write_pkt_lines_on_band(u3_atom band, u3_noun sea); + + u3_noun u3qe_git_pack_expand_delta_object(u3_noun, u3_noun); u3_noun u3qeo_raw(u3_atom, u3_atom); diff --git a/pkg/noun/jets/tree.c b/pkg/noun/jets/tree.c index 4940137d72..36361377f1 100644 --- a/pkg/noun/jets/tree.c +++ b/pkg/noun/jets/tree.c @@ -2147,28 +2147,26 @@ static u3j_core _139_hex_stream_d[] = }; /* - * Git protocol + * Git HTTP protocol */ -static u3j_harm _139_hex_git_io_stream_pkt_lines_on_band_a[] = - {{".2", u3we_git_io_stream_pkt_lines_on_band, c3y}, {}}; +static u3j_harm _139_hex_git_http_read_pkt_lines_on_band_a[] = + {{".2", u3we_git_http_read_pkt_lines_on_band, c3y}, {}}; +static u3j_harm _139_hex_git_http_write_pkt_lines_on_band_a[] = + {{".2", u3we_git_http_write_pkt_lines_on_band, c3y}, {}}; -static u3j_core _139_hex_git_io_d[] = +static u3j_core _139_hex_git_http_d[] = { - { "stream-pkt-lines-on-band", 7, _139_hex_git_io_stream_pkt_lines_on_band_a, 0, no_hashes}, + { "read-pkt-lines-on-band", 7, _139_hex_git_http_read_pkt_lines_on_band_a, 0, no_hashes}, + { "write-pkt-lines-on-band", 7, _139_hex_git_http_write_pkt_lines_on_band_a, 0, no_hashes}, {} }; -static u3j_harm _139_hex_git_pak_expand_delta_object_a[] = - {{".2", u3we_git_pak_expand_delta_object, c3y}, {}}; +static u3j_harm _139_hex_git_pack_expand_delta_object_a[] = + {{".2", u3we_git_pack_expand_delta_object, c3y}, {}}; -static u3j_core _139_hex_git_pak_d[] = - {{"expand-delta-object", 7, _139_hex_git_pak_expand_delta_object_a, c3y}, {}}; +static u3j_core _139_hex_git_pack_d[] = + {{"expand-delta-object", 7, _139_hex_git_pack_expand_delta_object_a, c3y}, {}}; -static u3j_core _139_hex_git_d[] = - { - {"pak", 7, 0, _139_hex_git_pak_d, no_hashes}, - {} - }; static u3j_core _139_hex_d[] = { { "lore", 63, _140_hex_lore_a, 0, no_hashes }, { "leer", 63, _140_hex_leer_a, 0, no_hashes }, @@ -2189,8 +2187,8 @@ static u3j_core _139_hex_d[] = { "json", 31, 0, _139_hex_json_d, no_hashes }, { "zlib", 31, 0, _139_hex_zlib_d, no_hashes}, { "stream", 31, 0, _139_hex_stream_d, no_hashes}, - { "git" , 31, 0, _139_hex_git_d, no_hashes}, - { "git-io", 31, 0, _139_hex_git_io_d, no_hashes}, + { "git-pack" , 31, 0, _139_hex_git_pack_d, no_hashes}, + { "git-http", 31, 0, _139_hex_git_http_d, no_hashes}, {} }; diff --git a/pkg/noun/jets/w.h b/pkg/noun/jets/w.h index 3a72856b71..ae58b224e3 100644 --- a/pkg/noun/jets/w.h +++ b/pkg/noun/jets/w.h @@ -231,8 +231,10 @@ u3_noun u3we_stream_append_get_bytes(u3_noun); u3_noun u3we_stream_append_read_bytes(u3_noun); - u3_noun u3we_git_io_stream_pkt_lines_on_band(u3_noun); - u3_noun u3we_git_pak_expand_delta_object(u3_noun); + u3_noun u3we_git_http_read_pkt_lines_on_band(u3_noun); + u3_noun u3we_git_http_write_pkt_lines_on_band(u3_noun); + + u3_noun u3we_git_pack_expand_delta_object(u3_noun); u3_noun u3we_bend_fun(u3_noun); u3_noun u3we_cold_fun(u3_noun); From e3c489adcf088c071270c78ea41b86dc45065599 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miko=C5=82aj=20Paraniak?= Date: Thu, 4 Apr 2024 14:04:53 +0800 Subject: [PATCH 09/12] jets: move git jets to %138 declarations --- pkg/noun/jets/tree.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/pkg/noun/jets/tree.c b/pkg/noun/jets/tree.c index 36361377f1..e2f36392db 100644 --- a/pkg/noun/jets/tree.c +++ b/pkg/noun/jets/tree.c @@ -2187,7 +2187,7 @@ static u3j_core _139_hex_d[] = { "json", 31, 0, _139_hex_json_d, no_hashes }, { "zlib", 31, 0, _139_hex_zlib_d, no_hashes}, { "stream", 31, 0, _139_hex_stream_d, no_hashes}, - { "git-pack" , 31, 0, _139_hex_git_pack_d, no_hashes}, + { "git-pack", 31, 0, _139_hex_git_pack_d, no_hashes}, { "git-http", 31, 0, _139_hex_git_http_d, no_hashes}, {} }; @@ -2410,6 +2410,11 @@ static u3j_core _138_hex_d[] = { "secp", 6, 0, _140_hex_secp_d, no_hashes }, { "mimes", 31, 0, _140_hex_mimes_d, no_hashes }, { "json", 31, 0, _139_hex_json_d, no_hashes }, + /* Git jets */ + { "zlib", 31, 0, _139_hex_zlib_d, no_hashes}, + { "stream", 31, 0, _139_hex_stream_d, no_hashes}, + { "git-pack", 31, 0, _139_hex_git_pack_d, no_hashes}, + { "git-http", 31, 0, _139_hex_git_http_d, no_hashes}, {} }; From 4e0e4678fd2d5f7ec619000315606c53e3f334fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miko=C5=82aj=20Paraniak?= Date: Sat, 13 Apr 2024 16:18:59 +0800 Subject: [PATCH 10/12] git jets: do not handle %ref-delta, as it requires object lookup --- pkg/noun/jets/e/git.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pkg/noun/jets/e/git.c b/pkg/noun/jets/e/git.c index 8c7d7d0856..25386d10ff 100644 --- a/pkg/noun/jets/e/git.c +++ b/pkg/noun/jets/e/git.c @@ -128,8 +128,7 @@ u3_noun u3qe_git_pack_expand_delta_object(u3_noun base, if ( c3y == u3r_sing(delta_type, REF_DELTA) ) { // fprintf(stderr, "Expanding %%ref-delta object\r\n"); - delta_octs = delta_obj; - u3_assert(false); + return u3_none; } u3x_cell(delta_octs, &delta_p_octs, &delta_q_octs); From 6125c1cd84b77886f7ffd1c785b963f5006ff8b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miko=C5=82aj=20Paraniak?= Date: Tue, 30 Apr 2024 15:16:42 +0800 Subject: [PATCH 11/12] git jets: fix an error in +expand-delta-object --- pkg/noun/jets/e/git.c | 53 +++++++++++++++++++++++++++---------------- 1 file changed, 34 insertions(+), 19 deletions(-) diff --git a/pkg/noun/jets/e/git.c b/pkg/noun/jets/e/git.c index 25386d10ff..4899268aa6 100644 --- a/pkg/noun/jets/e/git.c +++ b/pkg/noun/jets/e/git.c @@ -7,11 +7,28 @@ #include "allocate.h" #include "noun.h" -#define OFS_DELTA (u3i_string("ofs-delta")) -#define REF_DELTA (u3i_string("ref-delta")) - static u3_atom _q_octs; +static inline c3_y _cord_cmp(u3_atom cord, const char* s) +{ + u3a_atom* cord_p = u3a_to_ptr(cord); + const c3_y* c = (c3_y*)cord_p->buf_w; + c3_w len_w = u3r_met(3, cord); + + if (len_w == 0 || *s == '\0' ) { + return (len_w == 0 && *s == '\0') ? c3y : c3n; + } + + do { + if (*(c++) != *(s++)) { + return c3n; + } + + } while (--len_w && *s != '\0'); + + return (len_w == 0 && *s == '\0') ? c3y : c3n; +} + static c3_y* _unpack_octs(u3_atom p_octs, u3_atom* q_octs_p, c3_w* len_wp) { c3_y* buf_y; @@ -83,14 +100,13 @@ static c3_w _read_size(c3_y* buf_y, c3_w* pos_wp, c3_w buf_len_w, c3_w len_w) { } u3_noun u3qe_git_pack_expand_delta_object(u3_noun base, - u3_noun delta) { + u3_noun delta) { - /* +$ raw-object [type=object-type data=stream:libstream] - */ - /* +$ pack-object $% raw-object - [%ofs-delta pos=@ud base-offset=@ud =octs] - [%ref-delta =octs] - == + /* +$ raw-object [type=object-type size=@ud data=stream:libstream] + +$ pack-object $% raw-object + [%ofs-delta pos=@ud base-offset=@ud =octs] + [%ref-delta pos=@ud =hash =octs] + == +$ pack-delta-object $>(?(%ofs-delta %ref-delta) pack-object) */ @@ -109,7 +125,7 @@ u3_noun u3qe_git_pack_expand_delta_object(u3_noun base, // delta=pack-object // - u3_noun delta_type; + u3_atom delta_type; u3_noun delta_obj; u3_atom delta_pos; @@ -121,15 +137,12 @@ u3_noun u3qe_git_pack_expand_delta_object(u3_noun base, u3x_cell(delta, &delta_type, &delta_obj); - if ( c3y == u3r_sing(delta_type, OFS_DELTA) ) { - // fprintf(stderr, "Expanding %%ofs-delta object\r\n"); - u3x_trel(delta_obj, &delta_pos, &delta_base_offset, &delta_octs); - } - - if ( c3y == u3r_sing(delta_type, REF_DELTA) ) { - // fprintf(stderr, "Expanding %%ref-delta object\r\n"); + if (c3y == _cord_cmp(delta_type, "ref-delta")) { return u3_none; } + if (c3y == _cord_cmp(delta_type, "ofs-delta")) { + u3x_trel(delta_obj, &delta_pos, &delta_base_offset, &delta_octs); + } u3x_cell(delta_octs, &delta_p_octs, &delta_q_octs); @@ -323,7 +336,9 @@ u3_noun u3qe_git_pack_expand_delta_object(u3_noun base, } u3_noun data = u3nc(0, u3nc(u3i_chub(siz_w), u3i_slab_mint(&sab_u))); - u3_noun rob = u3nt(u3k(base_type), u3k(base_size), data); + u3_noun rob = u3nt(u3k(base_type), u3i_chub(siz_w), data); + + // fprintf(stderr, "Resolved object of size %d, siz_w = %d\r\n", base_size, siz_w); return rob; } From 47b27101fbeb878ce447b8ff09dd24e036d75d73 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miko=C5=82aj=20Paraniak?= Date: Mon, 22 Jul 2024 11:47:27 +0800 Subject: [PATCH 12/12] git jets: switch to libbytestream --- pkg/noun/jets/e/git.c | 17 +++--- pkg/noun/jets/e/git_http.c | 14 ++--- pkg/noun/jets/e/zlib.c | 27 ++++++---- pkg/noun/jets/q.h | 2 +- pkg/noun/jets/tree.c | 105 ++++++++++++++++++------------------- 5 files changed, 83 insertions(+), 82 deletions(-) diff --git a/pkg/noun/jets/e/git.c b/pkg/noun/jets/e/git.c index 4899268aa6..7368c9af76 100644 --- a/pkg/noun/jets/e/git.c +++ b/pkg/noun/jets/e/git.c @@ -102,7 +102,9 @@ static c3_w _read_size(c3_y* buf_y, c3_w* pos_wp, c3_w buf_len_w, c3_w len_w) { u3_noun u3qe_git_pack_expand_delta_object(u3_noun base, u3_noun delta) { - /* +$ raw-object [type=object-type size=@ud data=stream:libstream] + // +$ raw-object [type=object-type size=@ud data=octs] + + /* +$ raw-object [type=object-type size=@ud data=octs] +$ pack-object $% raw-object [%ofs-delta pos=@ud base-offset=@ud =octs] [%ref-delta pos=@ud =hash =octs] @@ -116,12 +118,11 @@ u3_noun u3qe_git_pack_expand_delta_object(u3_noun base, u3_atom base_size; u3_noun base_data; - u3_atom base_data_pos; u3_atom base_data_p_octs; u3_atom base_data_q_octs; u3x_trel(base, &base_type, &base_size, &base_data); - u3x_trel(base_data, &base_data_pos, &base_data_p_octs, &base_data_q_octs); + u3x_cell(base_data, &base_data_p_octs, &base_data_q_octs); // delta=pack-object // @@ -165,7 +166,7 @@ u3_noun u3qe_git_pack_expand_delta_object(u3_noun base, c3_w bas_len_w; bas_y = _unpack_octs(base_data_p_octs, &base_data_q_octs, &bas_buf_len_w); - // u3_assert(base_size == (base_data_p_octs - base_data_pos)); + u3_assert(base_size == base_data_p_octs); bas_begin_y = bas_y; bas_len_w = base_data_p_octs; @@ -180,11 +181,9 @@ u3_noun u3qe_git_pack_expand_delta_object(u3_noun base, // Base size mismatch // - if (biz_w != (base_data_p_octs - base_data_pos)) { + if (biz_w != base_data_p_octs) { fprintf(stderr, "bas_buf_len_w = %d, bas_len_w = %d\r\n", bas_buf_len_w, bas_len_w); fprintf(stderr, "sea_pos = %d, sea_buf_len = %d, sea_len = %d\r\n", sea_pos_w, sea_buf_len_w, sea_len_w); - fprintf(stderr, "u3qe_git_pack_expand_delta_object: base (pos = %d) object size mismatch!\r\n", base_data_pos); - // u3_assert(false); // _free(); return u3_none; @@ -335,11 +334,9 @@ u3_noun u3qe_git_pack_expand_delta_object(u3_noun base, return u3_none; } - u3_noun data = u3nc(0, u3nc(u3i_chub(siz_w), u3i_slab_mint(&sab_u))); + u3_noun data = u3nc(u3i_chub(siz_w), u3i_slab_mint(&sab_u)); u3_noun rob = u3nt(u3k(base_type), u3i_chub(siz_w), data); - // fprintf(stderr, "Resolved object of size %d, siz_w = %d\r\n", base_size, siz_w); - return rob; } diff --git a/pkg/noun/jets/e/git_http.c b/pkg/noun/jets/e/git_http.c index 523bcbbbc7..1280437ef9 100644 --- a/pkg/noun/jets/e/git_http.c +++ b/pkg/noun/jets/e/git_http.c @@ -45,8 +45,8 @@ static c3_y _to_hex_dit(c3_w byt) } u3_noun u3qe_git_http_read_pkt_lines_on_band( - u3_noun sea, - u3_atom band){ + u3_atom band, + u3_noun sea) { if (c3n == u3a_is_cat(band)) { u3m_bail(c3__fail); @@ -165,9 +165,9 @@ u3_noun u3qe_git_http_read_pkt_lines_on_band( sea_y += pkt_len_w; pos_w += pkt_len_w; } + fprintf(stderr, "Assembled %d bytes\r\n", total_w); u3_noun octs_red = u3nc(u3i_word(total_w), u3i_slab_mint(&sab_u)); - - return u3nc(u3nc(u3i_word(0), octs_red), u3k(sea)); + return u3nc(octs_red, u3k(sea)); } u3_noun u3we_git_http_read_pkt_lines_on_band(u3_noun cor) { @@ -280,8 +280,8 @@ u3_noun u3we_git_http_write_pkt_lines_on_band(u3_noun cor) { u3_noun sea; u3_atom band; - u3x_mean(cor, u3x_sam_2, &sea, - u3x_sam_3, &band, 0); + u3x_mean(cor, u3x_sam_2, &band, + u3x_sam_3, &sea, 0); - return u3qe_git_http_write_pkt_lines_on_band(sea, band); + return u3qe_git_http_write_pkt_lines_on_band(band, sea); } diff --git a/pkg/noun/jets/e/zlib.c b/pkg/noun/jets/e/zlib.c index 9d3489de13..c82bc7540e 100644 --- a/pkg/noun/jets/e/zlib.c +++ b/pkg/noun/jets/e/zlib.c @@ -14,21 +14,25 @@ #define OUTBUF_SZ 4096 -u3_noun u3qe_zlib_expand(u3_atom pos, u3_noun byts) { +u3_noun u3qe_zlib_expand(u3_atom pos, u3_atom oft, u3_noun octs) { c3_d pos_d; + c3_d oft_d; c3_d wid_d; u3_atom wid; u3_atom dat; - u3x_cell(byts, &wid, &dat); + u3x_cell(octs, &wid, &dat); size_t sad_i = u3r_met(3, dat); if (c3n == (u3r_safe_chub(pos, &pos_d))) { return u3_none; } + if (c3n == (u3r_safe_chub(oft, &oft_d))) { + return u3_none; + } if (c3n == (u3r_safe_chub(wid, &wid_d))) { return u3_none; } @@ -46,13 +50,14 @@ u3_noun u3qe_zlib_expand(u3_atom pos, u3_noun byts) { z_stream zea; c3_w zas_w; - zea.next_in = byt_y + pos_d; - zea.avail_in = (wid_d - pos_d); + zea.next_in = byt_y + oft_d + pos_d; + zea.avail_in = (wid_d - oft_d - pos_d); zea.zalloc = Z_NULL; zea.zfree = Z_NULL; zea.opaque = Z_NULL; + /* Allocate output buffer */ @@ -95,6 +100,7 @@ u3_noun u3qe_zlib_expand(u3_atom pos, u3_noun byts) { } } + size_t len_i = sob_i - zea.avail_out; pos_d += zea.total_in; @@ -112,15 +118,18 @@ u3_noun u3qe_zlib_expand(u3_atom pos, u3_noun byts) { u3_atom buf_a = u3i_slab_mint(&sab_u); return u3nc(u3nc(len_a, buf_a), - u3nc(u3i_chub(pos_d), u3k(byts))); + u3nt(u3i_chub(pos_d), u3k(oft), u3k(octs))); } -u3_noun u3we_zlib_expand(u3_noun sea) { +u3_noun u3we_zlib_expand(u3_noun cor) { u3_atom pos; - u3_noun byts; + u3_atom oft; + u3_noun octs; - u3x_mean(sea, u3x_sam_2, &pos, u3x_sam_3, &byts, 0); + u3x_mean(cor, u3x_sam_2, &pos, + u3x_sam_6, &oft, + u3x_sam_7, &octs, 0); - return u3qe_zlib_expand(pos, byts); + return u3qe_zlib_expand(pos, oft, octs); } diff --git a/pkg/noun/jets/q.h b/pkg/noun/jets/q.h index 9dc47b054a..c0bef1a27b 100644 --- a/pkg/noun/jets/q.h +++ b/pkg/noun/jets/q.h @@ -159,7 +159,7 @@ u3_noun u3qe_json_de(u3_atom); u3_atom u3qe_json_en(u3_noun); - u3_noun u3qe_zlib_expand(u3_atom, u3_noun); + u3_noun u3qe_zlib_expand(u3_atom, u3_atom, u3_noun); u3_noun u3qe_stream_append_get_bytes(u3_atom n, u3_noun red, u3_noun sea); u3_noun u3qe_stream_append_read_bytes(u3_atom n, u3_noun red, u3_noun sea); diff --git a/pkg/noun/jets/tree.c b/pkg/noun/jets/tree.c index e2f36392db..3b148b25ca 100644 --- a/pkg/noun/jets/tree.c +++ b/pkg/noun/jets/tree.c @@ -2120,53 +2120,6 @@ static u3j_core _139_hex_json_d[] = {} }; -/* - * Zlib - */ -static u3j_harm _139_hex_zlib_expand_a[] = {{".2", u3we_zlib_expand, c3y}, {}}; -static u3j_core _139_hex_zlib_d[] = - { - { "expand", 7, _139_hex_zlib_expand_a, 0, no_hashes }, - {} - }; - -/* - * Stream library - */ -static u3j_harm _139_hex_stream_append_get_bytes_a[] = - {{".2", u3we_stream_append_get_bytes, c3y},{}}; - -static u3j_harm _139_hex_stream_append_read_bytes_a[] = - {{".2", u3we_stream_append_read_bytes, c3y},{}}; - -static u3j_core _139_hex_stream_d[] = - { - { "append-get-bytes", 7, _139_hex_stream_append_get_bytes_a, 0, no_hashes }, - { "append-read-bytes", 7, _139_hex_stream_append_read_bytes_a, 0, no_hashes }, - {} - }; - -/* - * Git HTTP protocol - */ -static u3j_harm _139_hex_git_http_read_pkt_lines_on_band_a[] = - {{".2", u3we_git_http_read_pkt_lines_on_band, c3y}, {}}; -static u3j_harm _139_hex_git_http_write_pkt_lines_on_band_a[] = - {{".2", u3we_git_http_write_pkt_lines_on_band, c3y}, {}}; - -static u3j_core _139_hex_git_http_d[] = - { - { "read-pkt-lines-on-band", 7, _139_hex_git_http_read_pkt_lines_on_band_a, 0, no_hashes}, - { "write-pkt-lines-on-band", 7, _139_hex_git_http_write_pkt_lines_on_band_a, 0, no_hashes}, - {} - }; - -static u3j_harm _139_hex_git_pack_expand_delta_object_a[] = - {{".2", u3we_git_pack_expand_delta_object, c3y}, {}}; - -static u3j_core _139_hex_git_pack_d[] = - {{"expand-delta-object", 7, _139_hex_git_pack_expand_delta_object_a, c3y}, {}}; - static u3j_core _139_hex_d[] = { { "lore", 63, _140_hex_lore_a, 0, no_hashes }, { "leer", 63, _140_hex_leer_a, 0, no_hashes }, @@ -2185,10 +2138,6 @@ static u3j_core _139_hex_d[] = { "secp", 6, 0, _140_hex_secp_d, no_hashes }, { "mimes", 31, 0, _140_hex_mimes_d, no_hashes }, { "json", 31, 0, _139_hex_json_d, no_hashes }, - { "zlib", 31, 0, _139_hex_zlib_d, no_hashes}, - { "stream", 31, 0, _139_hex_stream_d, no_hashes}, - { "git-pack", 31, 0, _139_hex_git_pack_d, no_hashes}, - { "git-http", 31, 0, _139_hex_git_http_d, no_hashes}, {} }; @@ -2391,6 +2340,52 @@ static u3j_core _138_hex_blake_d[] = { "blake3-impl", 7, 0, _138_hex_blake3_impl_d, no_hashes }, {} }; +/* + * Zlib + */ +static u3j_harm _138_hex_zlib_expand_a[] = {{".2", u3we_zlib_expand, c3y}, {}}; +static u3j_core _138_hex_zlib_d[] = + { + { "expand", 7, _138_hex_zlib_expand_a, 0, no_hashes }, + {} + }; + +/* + * Stream library + */ +static u3j_harm _138_hex_stream_append_get_bytes_a[] = + {{".2", u3we_stream_append_get_bytes, c3y},{}}; + +static u3j_harm _138_hex_stream_append_read_bytes_a[] = + {{".2", u3we_stream_append_read_bytes, c3y},{}}; + +static u3j_core _138_hex_stream_d[] = + { + { "append-get-bytes", 7, _138_hex_stream_append_get_bytes_a, 0, no_hashes }, + { "append-read-bytes", 7, _138_hex_stream_append_read_bytes_a, 0, no_hashes }, + {} + }; + +/* + * Git HTTP protocol + */ +static u3j_harm _138_hex_git_http_read_pkt_lines_on_band_a[] = + {{".2", u3we_git_http_read_pkt_lines_on_band, c3y}, {}}; +static u3j_harm _138_hex_git_http_write_pkt_lines_on_band_a[] = + {{".2", u3we_git_http_write_pkt_lines_on_band, c3y}, {}}; + +static u3j_core _138_hex_git_http_d[] = + { + { "read-pkt-lines-on-band", 7, _138_hex_git_http_read_pkt_lines_on_band_a, 0, no_hashes}, + { "write-pkt-lines-on-band", 7, _138_hex_git_http_write_pkt_lines_on_band_a, 0, no_hashes}, + {} + }; + +static u3j_harm _138_hex_git_pack_expand_delta_object_a[] = + {{".2", u3we_git_pack_expand_delta_object, c3y}, {}}; + +static u3j_core _138_hex_git_pack_d[] = + {{"expand-delta-object", 7, _138_hex_git_pack_expand_delta_object_a, c3y}, {}}; static u3j_core _138_hex_d[] = { { "lore", 63, _140_hex_lore_a, 0, no_hashes }, @@ -2411,10 +2406,10 @@ static u3j_core _138_hex_d[] = { "mimes", 31, 0, _140_hex_mimes_d, no_hashes }, { "json", 31, 0, _139_hex_json_d, no_hashes }, /* Git jets */ - { "zlib", 31, 0, _139_hex_zlib_d, no_hashes}, - { "stream", 31, 0, _139_hex_stream_d, no_hashes}, - { "git-pack", 31, 0, _139_hex_git_pack_d, no_hashes}, - { "git-http", 31, 0, _139_hex_git_http_d, no_hashes}, + { "zlib", 31, 0, _138_hex_zlib_d, no_hashes}, + { "stream", 31, 0, _138_hex_stream_d, no_hashes}, + { "git-pack", 31, 0, _138_hex_git_pack_d, no_hashes}, + { "git-http", 31, 0, _138_hex_git_http_d, no_hashes}, {} };