Turbopack: Add bincode encode/decode wrappers for serde_json::Value when stored in a cell

Author: bgw

Cargo.lock

@@ -1,22 +1,19 @@
-use proc_macro2::Span;
 use syn::{
-    Meta, Result, Token, Type,
+    MacroDelimiter, Meta, MetaList, Result, Token, Type,
     parse::{Parse, ParseStream},
-    spanned::Spanned,
 };
 
-#[derive(Debug)]
 pub struct PrimitiveInput {
     pub ty: Type,
-    pub manual_shrink_to_fit: Option<Span>,
+    pub bincode_wrappers: Option<BincodeWrappers>,
 }
 
 impl Parse for PrimitiveInput {
     fn parse(input: ParseStream) -> Result<Self> {
         let ty: Type = input.parse()?;
         let mut parsed_input = PrimitiveInput {
             ty,
-            manual_shrink_to_fit: None,
+            bincode_wrappers: None,
         };
         if input.parse::<Option<Token![,]>>()?.is_some() {
             let punctuated = input.parse_terminated(Meta::parse, Token![,])?;
@@ -27,15 +24,26 @@ impl Parse for PrimitiveInput {
                         .map(ToString::to_string)
                         .as_deref()
                         .unwrap_or_default(),
-                    &meta,
+                    meta,
                 ) {
-                    ("manual_shrink_to_fit", Meta::Path(_)) => {
-                        parsed_input.manual_shrink_to_fit = Some(meta.span())
+                    ("bincode_wrappers", meta) => {
+                        let Meta::List(MetaList {
+                            tokens: wrapper_tokens,
+                            delimiter: MacroDelimiter::Paren(..),
+                            ..
+                        }) = meta
+                        else {
+                            return Err(syn::Error::new_spanned(
+                                meta,
+                                "expected parenthesized (EncodeTy, DecodeTy) list",
+                            ));
+                        };
+                        parsed_input.bincode_wrappers = Some(syn::parse2(wrapper_tokens)?);
                     }
                     (_, meta) => {
                         return Err(syn::Error::new_spanned(
                             meta,
-                            "unexpected token, expected: \"manual_shrink_to_fit\"",
+                            "unexpected token, expected: \"bincode_wrappers\"",
                         ));
                     }
                 }
@@ -44,3 +52,28 @@ impl Parse for PrimitiveInput {
         Ok(parsed_input)
     }
 }
+
+// TODO: wire this up in https://github.com/vercel/next.js/pull/86338
+#[allow(dead_code)]
+pub struct BincodeWrappers {
+    pub encode_ty: Type,
+    pub decode_ty: Type,
+}
+
+impl Parse for BincodeWrappers {
+    fn parse(input: ParseStream) -> Result<Self> {
+        let punctuated = input.parse_terminated(Type::parse, Token![,])?;
+        let items: [Type; 2] = punctuated
+            .into_iter()
+            .collect::<Vec<_>>()
+            .try_into()
+            .map_err(|_| {
+                syn::Error::new(input.span(), "expected exactly two comma-separated types")
+            })?;
+        let (encode_ty, decode_ty) = items.into();
+        Ok(BincodeWrappers {
+            encode_ty,
+            decode_ty,
+        })
+    }
+}

turbopack/crates/turbo-tasks-macros/src/primitive_input.rs

@@ -8,9 +8,11 @@ use crate::{
 };
 
 pub fn primitive(input: TokenStream) -> TokenStream {
-    let input = parse_macro_input!(input as PrimitiveInput);
+    let PrimitiveInput {
+        ty,
+        bincode_wrappers: _,
+    } = parse_macro_input!(input as PrimitiveInput);
 
-    let ty = input.ty;
     let Some(ident) = get_type_ident(&ty) else {
         return quote! {
             // An error occurred while parsing the ident.
@@ -34,7 +36,13 @@ pub fn primitive(input: TokenStream) -> TokenStream {
             }
         }
     };
-    let name = global_name(quote! {stringify!(#ty) });
+
+    let name = global_name(quote!(stringify!(#ty)));
+    // TODO: https://github.com/vercel/next.js/pull/86338 -- switch to bincode, use bincode wrapper
+    let new_value_type = quote! {
+        turbo_tasks::ValueType::new_with_any_serialization::<#ty>(#name);
+    };
+
     let value_type_and_register = value_type_and_register(
         &ident,
         quote! { #ty },
@@ -45,10 +53,8 @@ pub fn primitive(input: TokenStream) -> TokenStream {
         quote! {
             turbo_tasks::VcCellCompareMode<#ty>
         },
-        quote! {
-            turbo_tasks::ValueType::new_with_any_serialization::<#ty>(#name)
-        },
-        quote! { true },
+        new_value_type,
+        /* has_serialization */ quote! { true },
     );
 
     let value_default_impl = quote! {

turbopack/crates/turbo-tasks-macros/src/primitive_macro.rs

@@ -48,6 +48,7 @@ tokio = { workspace = true, features = ["full"] }
 tokio-util = { workspace = true }
 tracing = { workspace = true }
 triomphe = { workspace = true, features = ["unsize", "unstable"] }
+turbo-bincode = { workspace = true }
 turbo-dyn-eq-hash = { workspace = true }
 turbo-rcstr = { workspace = true }
 turbo-tasks-hash = { workspace = true }

turbopack/crates/turbo-tasks/Cargo.toml

@@ -1,18 +1,25 @@
 use std::time::Duration;
 
+use bincode::{
+    Decode, Encode,
+    de::Decoder,
+    enc::Encoder,
+    error::{DecodeError, EncodeError},
+};
 use turbo_rcstr::RcStr;
 use turbo_tasks_macros::primitive as __turbo_tasks_internal_primitive;
 
 use crate::{
-    Vc, {self as turbo_tasks},
+    self as turbo_tasks, Vc,
+    value_type::{ManualDecodeWrapper, ManualEncodeWrapper},
 };
 
 __turbo_tasks_internal_primitive!(());
-__turbo_tasks_internal_primitive!(String, manual_shrink_to_fit);
+__turbo_tasks_internal_primitive!(String);
 __turbo_tasks_internal_primitive!(RcStr);
 __turbo_tasks_internal_primitive!(Option<String>);
 __turbo_tasks_internal_primitive!(Option<RcStr>);
-__turbo_tasks_internal_primitive!(Vec<RcStr>, manual_shrink_to_fit);
+__turbo_tasks_internal_primitive!(Vec<RcStr>);
 __turbo_tasks_internal_primitive!(Option<u16>);
 __turbo_tasks_internal_primitive!(Option<u64>);
 __turbo_tasks_internal_primitive!(bool);
@@ -29,7 +36,46 @@ __turbo_tasks_internal_primitive!(i64);
 __turbo_tasks_internal_primitive!(i128);
 __turbo_tasks_internal_primitive!(usize);
 __turbo_tasks_internal_primitive!(isize);
-__turbo_tasks_internal_primitive!(serde_json::Value);
+__turbo_tasks_internal_primitive!(
+    serde_json::Value,
+    bincode_wrappers(JsonValueEncodeWrapper, JsonValueDecodeWrapper),
+);
 __turbo_tasks_internal_primitive!(Duration);
-__turbo_tasks_internal_primitive!(Vec<u8>, manual_shrink_to_fit);
-__turbo_tasks_internal_primitive!(Vec<bool>, manual_shrink_to_fit);
+__turbo_tasks_internal_primitive!(Vec<u8>);
+__turbo_tasks_internal_primitive!(Vec<bool>);
+
+// TODO: use this in https://github.com/vercel/next.js/pull/86338
+#[allow(dead_code)]
+struct JsonValueEncodeWrapper<'a>(&'a serde_json::Value);
+
+impl ManualEncodeWrapper for JsonValueEncodeWrapper<'_> {
+    type Value = serde_json::Value;
+
+    fn new<'a>(value: &'a Self::Value) -> impl Encode + 'a {
+        JsonValueEncodeWrapper(value)
+    }
+}
+
+impl Encode for JsonValueEncodeWrapper<'_> {
+    fn encode<E: Encoder>(&self, encoder: &mut E) -> Result<(), EncodeError> {
+        turbo_bincode::serde_json::encode(self.0, encoder)
+    }
+}
+
+// TODO: use this in https://github.com/vercel/next.js/pull/86338
+#[allow(dead_code)]
+struct JsonValueDecodeWrapper(serde_json::Value);
+
+impl ManualDecodeWrapper for JsonValueDecodeWrapper {
+    type Value = serde_json::Value;
+
+    fn inner(self) -> Self::Value {
+        self.0
+    }
+}
+
+impl<Context> Decode<Context> for JsonValueDecodeWrapper {
+    fn decode<D: Decoder<Context = Context>>(decoder: &mut D) -> Result<Self, DecodeError> {
+        Ok(Self(turbo_bincode::serde_json::decode(decoder)?))
+    }
+}

turbopack/crates/turbo-tasks/src/primitives.rs

@@ -5,6 +5,7 @@ use std::{
 };
 
 use auto_hash_map::{AutoMap, AutoSet};
+use bincode::{Decode, Encode};
 use serde::{Deserialize, Serialize};
 use tracing::Span;
 
@@ -98,6 +99,21 @@ pub fn any_as_serialize<T: Any + Serialize + Send + Sync + 'static>(
     );
 }
 
+// TODO: use this in https://github.com/vercel/next.js/pull/86338
+#[allow(dead_code)]
+pub trait ManualEncodeWrapper: Encode {
+    type Value;
+    // this uses RPIT to avoid some lifetime problems
+    fn new<'a>(value: &'a Self::Value) -> impl Encode + 'a;
+}
+
+// TODO: use this in https://github.com/vercel/next.js/pull/86338
+#[allow(dead_code)]
+pub trait ManualDecodeWrapper: Decode<()> {
+    type Value;
+    fn inner(self) -> Self::Value;
+}
+
 impl ValueType {
     /// This is internally used by `#[turbo_tasks::value]`
     pub fn new<T: VcValueType>(global_name: &'static str) -> Self {