diff --git a/Cargo.lock b/Cargo.lock index 6dc53a67fa519..5823ff9081cb8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9238,7 +9238,6 @@ dependencies = [ "turbo-tasks-macros", "turbo-tasks-malloc", "unsize", - "unty", ] [[package]] @@ -9288,6 +9287,7 @@ dependencies = [ "afl", "anyhow", "arbitrary", + "bincode 2.0.1", "libfuzzer-sys", "once_cell", "serde", @@ -9426,6 +9426,7 @@ name = "turbo-tasks-macros-tests" version = "0.1.0" dependencies = [ "anyhow", + "bincode 2.0.1", "serde", "tokio", "trybuild", @@ -9566,6 +9567,7 @@ name = "turbopack-cli" version = "0.1.0" dependencies = [ "anyhow", + "bincode 2.0.1", "clap", "codspeed-criterion-compat", "console-subscriber", diff --git a/crates/next-api/src/operation.rs b/crates/next-api/src/operation.rs index 316939bf1a46f..69db45d689276 100644 --- a/crates/next-api/src/operation.rs +++ b/crates/next-api/src/operation.rs @@ -123,6 +123,8 @@ fn pick_route(entrypoints: OperationVc, key: RcStr, route: &Route) ValueDebugFormat, NonLocalValue, OperationValue, + Encode, + Decode, )] enum EndpointSelector { RoutePageHtml(RcStr), diff --git a/crates/next-api/src/pages.rs b/crates/next-api/src/pages.rs index 5ff3a9b596a1f..9c3e4b600fd03 100644 --- a/crates/next-api/src/pages.rs +++ b/crates/next-api/src/pages.rs @@ -612,7 +612,18 @@ enum PageEndpointType { } #[derive( - Copy, Clone, Serialize, Deserialize, PartialEq, Eq, Hash, Debug, TaskInput, TraceRawVcs, + Copy, + Clone, + Serialize, + Deserialize, + PartialEq, + Eq, + Hash, + Debug, + TaskInput, + TraceRawVcs, + Encode, + Decode, )] enum SsrChunkType { Page, @@ -621,7 +632,18 @@ enum SsrChunkType { } #[derive( - Copy, Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize, TaskInput, TraceRawVcs, + Copy, + Clone, + Debug, + PartialEq, + Eq, + Hash, + Serialize, + Deserialize, + TaskInput, + TraceRawVcs, + Encode, + Decode, )] enum EmitManifests { /// Don't emit any manifests diff --git a/crates/next-api/src/project.rs b/crates/next-api/src/project.rs index 5c4737815b562..d686ee592944d 100644 --- a/crates/next-api/src/project.rs +++ b/crates/next-api/src/project.rs @@ -209,10 +209,6 @@ pub struct ProjectOptions { pub current_node_js_version: RcStr, } -#[derive( - Debug, Serialize, Deserialize, Clone, TaskInput, PartialEq, Eq, Hash, TraceRawVcs, NonLocalValue, -)] -#[serde(rename_all = "camelCase")] pub struct PartialProjectOptions { /// A root path from which all files must be nested under. Trying to access /// a file outside this root will fail. Think of this as a chroot. diff --git a/crates/next-core/src/next_client/context.rs b/crates/next-core/src/next_client/context.rs index 5861905a72101..22746215bc9b8 100644 --- a/crates/next-core/src/next_client/context.rs +++ b/crates/next-core/src/next_client/context.rs @@ -1,6 +1,7 @@ use std::collections::BTreeSet; use anyhow::Result; +use bincode::{Decode, Encode}; use serde::{Deserialize, Serialize}; use turbo_rcstr::{RcStr, rcstr}; use turbo_tasks::{ResolvedVc, TaskInput, Vc, trace::TraceRawVcs}; @@ -32,11 +33,13 @@ use turbopack_node::{ }; use turbopack_resolve::resolve_options_context::ResolveOptionsContext; -use super::transforms::get_next_client_transforms_rules; use crate::{ mode::NextMode, next_build::get_postcss_package_mapping, - next_client::runtime_entry::{RuntimeEntries, RuntimeEntry}, + next_client::{ + runtime_entry::{RuntimeEntries, RuntimeEntry}, + transforms::get_next_client_transforms_rules, + }, next_config::NextConfig, next_font::local::NextFontLocalResolvePlugin, next_import_map::{ @@ -405,7 +408,19 @@ pub async fn get_client_module_options_context( Ok(module_options_context) } -#[derive(Clone, Debug, PartialEq, Eq, Hash, TaskInput, TraceRawVcs, Serialize, Deserialize)] +#[derive( + Clone, + Debug, + PartialEq, + Eq, + Hash, + TaskInput, + TraceRawVcs, + Serialize, + Deserialize, + Encode, + Decode, +)] pub struct ClientChunkingContextOptions { pub mode: Vc, pub root_path: FileSystemPath, diff --git a/crates/next-core/src/next_edge/context.rs b/crates/next-core/src/next_edge/context.rs index ffecca7e4259c..347dcf1924b09 100644 --- a/crates/next-core/src/next_edge/context.rs +++ b/crates/next-core/src/next_edge/context.rs @@ -1,4 +1,5 @@ use anyhow::Result; +use bincode::{Decode, Encode}; use serde::{Deserialize, Serialize}; use turbo_rcstr::{RcStr, rcstr}; use turbo_tasks::{ResolvedVc, TaskInput, Vc, trace::TraceRawVcs}; @@ -196,7 +197,19 @@ pub async fn get_edge_resolve_options_context( .cell()) } -#[derive(Clone, Debug, PartialEq, Eq, Hash, TaskInput, TraceRawVcs, Serialize, Deserialize)] +#[derive( + Clone, + Debug, + PartialEq, + Eq, + Hash, + TaskInput, + TraceRawVcs, + Serialize, + Deserialize, + Encode, + Decode, +)] pub struct EdgeChunkingContextOptions { pub mode: Vc, pub root_path: FileSystemPath, diff --git a/crates/next-core/src/next_root_params/mod.rs b/crates/next-core/src/next_root_params/mod.rs index 667ab2dc15fcb..6cd37016a738f 100644 --- a/crates/next-core/src/next_root_params/mod.rs +++ b/crates/next-core/src/next_root_params/mod.rs @@ -5,7 +5,7 @@ use either::Either; use indoc::formatdoc; use itertools::Itertools; use turbo_rcstr::RcStr; -use turbo_tasks::{ResolvedVc, Vc}; +use turbo_tasks::{EitherTaskInput, ResolvedVc, Vc}; use turbo_tasks_fs::{FileContent, FileSystemPath}; use turbopack_core::{ asset::AssetContent, @@ -36,9 +36,13 @@ pub async fn insert_next_root_params_mapping( ) -> Result<()> { import_map.insert_exact_alias( "next/root-params", - get_next_root_params_mapping(is_root_params_enabled, ty, collected_root_params) - .to_resolved() - .await?, + get_next_root_params_mapping( + is_root_params_enabled, + EitherTaskInput(ty), + collected_root_params, + ) + .to_resolved() + .await?, ); Ok(()) } @@ -46,7 +50,7 @@ pub async fn insert_next_root_params_mapping( #[turbo_tasks::function] async fn get_next_root_params_mapping( is_root_params_enabled: Vc, - ty: Either, + ty: EitherTaskInput, collected_root_params: Option>, ) -> Result> { // This mapping goes into the global resolve options, so we want to avoid invalidating it if @@ -77,12 +81,12 @@ impl NextRootParamsMapper { #[turbo_tasks::function] pub fn new( is_root_params_enabled: ResolvedVc, - context_type: Either, + context_type: EitherTaskInput, collected_root_params: Option>, ) -> Vc { NextRootParamsMapper { is_root_params_enabled, - context_type, + context_type: context_type.0, collected_root_params, } .cell() diff --git a/crates/next-core/src/next_server/context.rs b/crates/next-core/src/next_server/context.rs index a10620ef5847b..e5c677f5c004d 100644 --- a/crates/next-core/src/next_server/context.rs +++ b/crates/next-core/src/next_server/context.rs @@ -1,6 +1,7 @@ use std::collections::BTreeSet; use anyhow::{Result, bail}; +use bincode::{Decode, Encode}; use serde::{Deserialize, Serialize}; use turbo_rcstr::{RcStr, rcstr}; use turbo_tasks::{ResolvedVc, TaskInput, Vc, trace::TraceRawVcs}; @@ -38,10 +39,6 @@ use turbopack_node::{ use turbopack_nodejs::NodeJsChunkingContext; use turbopack_resolve::resolve_options_context::ResolveOptionsContext; -use super::{ - resolve::ExternalCjsModulesResolvePlugin, - transforms::{get_next_server_internal_transforms_rules, get_next_server_transforms_rules}, -}; use crate::{ app_structure::CollectedRootParams, mode::NextMode, @@ -49,7 +46,10 @@ use crate::{ next_config::NextConfig, next_font::local::NextFontLocalResolvePlugin, next_import_map::{get_next_edge_and_server_fallback_import_map, get_next_server_import_map}, - next_server::resolve::ExternalPredicate, + next_server::{ + resolve::{ExternalCjsModulesResolvePlugin, ExternalPredicate}, + transforms::{get_next_server_internal_transforms_rules, get_next_server_transforms_rules}, + }, next_shared::{ resolve::{ ModuleFeatureReportResolvePlugin, NextExternalResolvePlugin, @@ -981,7 +981,19 @@ pub async fn get_server_module_options_context( Ok(module_options_context) } -#[derive(Clone, Debug, PartialEq, Eq, Hash, TaskInput, TraceRawVcs, Serialize, Deserialize)] +#[derive( + Clone, + Debug, + PartialEq, + Eq, + Hash, + TaskInput, + TraceRawVcs, + Serialize, + Deserialize, + Encode, + Decode, +)] pub struct ServerChunkingContextOptions { pub mode: Vc, pub root_path: FileSystemPath, diff --git a/crates/next-core/src/util.rs b/crates/next-core/src/util.rs index 8ac122bb46007..cc73e695269c2 100644 --- a/crates/next-core/src/util.rs +++ b/crates/next-core/src/util.rs @@ -59,7 +59,18 @@ pub fn defines(define_env: &FxIndexMap>) -> CompileTimeDefi } #[derive( - Debug, Clone, Copy, PartialEq, Eq, Hash, TaskInput, Serialize, Deserialize, TraceRawVcs, + Debug, + Clone, + Copy, + PartialEq, + Eq, + Hash, + TaskInput, + Serialize, + Deserialize, + TraceRawVcs, + Encode, + Decode, )] pub enum PathType { PagesPage, diff --git a/turbopack/crates/turbo-bincode/src/lib.rs b/turbopack/crates/turbo-bincode/src/lib.rs index e9e9a2544db45..af6b399e8f59d 100644 --- a/turbopack/crates/turbo-bincode/src/lib.rs +++ b/turbopack/crates/turbo-bincode/src/lib.rs @@ -1,7 +1,7 @@ #[doc(hidden)] pub mod macro_helpers; -use std::ptr::copy_nonoverlapping; +use std::{any::Any, ptr::copy_nonoverlapping}; use ::smallvec::SmallVec; use bincode::{ @@ -17,6 +17,8 @@ pub type TurboBincodeEncoder<'a> = EncoderImpl, bincode::config::Configuration>; pub type TurboBincodeDecoder<'a> = DecoderImpl, bincode::config::Configuration, ()>; +pub type AnyEncodeFn = fn(&dyn Any, &mut TurboBincodeEncoder<'_>) -> Result<(), EncodeError>; +pub type AnyDecodeFn = fn(&mut TurboBincodeDecoder<'_>) -> Result; fn new_turbo_bincode_encoder(buf: &mut TurboBincodeBuffer) -> TurboBincodeEncoder<'_> { EncoderImpl::new(TurboBincodeWriter::new(buf), TURBO_BINCODE_CONFIG) diff --git a/turbopack/crates/turbo-tasks-backend/fuzz/Cargo.toml b/turbopack/crates/turbo-tasks-backend/fuzz/Cargo.toml index 3cd225eb56e14..bdea8685817f7 100644 --- a/turbopack/crates/turbo-tasks-backend/fuzz/Cargo.toml +++ b/turbopack/crates/turbo-tasks-backend/fuzz/Cargo.toml @@ -11,6 +11,7 @@ cargo-fuzz = true afl = "0.15.18" anyhow = { workspace = true } arbitrary = { version = "1.4.1", features = ["derive"] } +bincode = { workspace = true } libfuzzer-sys = "0.4.9" once_cell = { workspace = true } serde = { workspace = true } diff --git a/turbopack/crates/turbo-tasks-backend/fuzz/src/graph.rs b/turbopack/crates/turbo-tasks-backend/fuzz/src/graph.rs index a54b8063dfc3e..d4f25897d920f 100644 --- a/turbopack/crates/turbo-tasks-backend/fuzz/src/graph.rs +++ b/turbopack/crates/turbo-tasks-backend/fuzz/src/graph.rs @@ -2,6 +2,7 @@ use std::sync::Arc; use anyhow::Result; use arbitrary::Arbitrary; +use bincode::{Decode, Encode}; use once_cell::sync::Lazy; use serde::{Deserialize, Serialize}; use turbo_tasks::{self, NonLocalValue, State, TaskInput, TurboTasks, Vc, trace::TraceRawVcs}; @@ -19,6 +20,8 @@ use turbo_tasks_malloc::TurboMalloc; Deserialize, TraceRawVcs, TaskInput, + Encode, + Decode, )] pub struct TaskReferenceSpec { task: u16, @@ -39,6 +42,8 @@ pub struct TaskReferenceSpec { Deserialize, TraceRawVcs, TaskInput, + Encode, + Decode, )] pub struct TaskSpec { references: Vec, diff --git a/turbopack/crates/turbo-tasks-backend/src/kv_backing_storage.rs b/turbopack/crates/turbo-tasks-backend/src/kv_backing_storage.rs index 6966a70d66b8a..d8a2cfac4963b 100644 --- a/turbopack/crates/turbo-tasks-backend/src/kv_backing_storage.rs +++ b/turbopack/crates/turbo-tasks-backend/src/kv_backing_storage.rs @@ -604,17 +604,6 @@ fn encode_task_type( buffer: &mut TurboBincodeBuffer, task_id: Option, ) -> Result<()> { - // DO NOT REMOVE THE `inline(never)` ATTRIBUTE! - // CachedTaskType's `Encode`/`Decode` implementations use `pot` internally for `TaskInput`s. - // TODO: remove `serde` and `pot`, make `TaskInput: Encode + Decode`. - // - // `pot` uses the pointer address of `&'static str` to deduplicate Symbols. - // If this function is inlined into multiple different callsites it might inline the Serialize - // implementation too, which can pull a `&'static str` from another crate into this crate. - // Since string deduplication between crates is not guaranteed, it can lead to behavior changes - // due to the pointer addresses. This can lead to lookup path and store path creating different - // serialization of the same task type, which breaks task cache lookups. - #[inline(never)] fn encode_once_into( task_type: &CachedTaskType, buffer: &mut TurboBincodeBuffer, diff --git a/turbopack/crates/turbo-tasks-backend/tests/trace_transient.rs b/turbopack/crates/turbo-tasks-backend/tests/trace_transient.rs index d863dd79e8e18..24e3329b8b1d5 100644 --- a/turbopack/crates/turbo-tasks-backend/tests/trace_transient.rs +++ b/turbopack/crates/turbo-tasks-backend/tests/trace_transient.rs @@ -2,7 +2,7 @@ #![feature(arbitrary_self_types_pointers)] use anyhow::Result; -use serde::{Deserialize, Serialize}; +use bincode::{Decode, Encode}; use turbo_tasks::{NonLocalValue, ResolvedVc, TaskInput, Vc, trace::TraceRawVcs}; use turbo_tasks_testing::{Registration, register, run_once_without_cache_check}; @@ -62,9 +62,7 @@ async fn read_incorrect_task_input_operation(value: IncorrectTaskInput) -> Resul /// Has an intentionally incorrect `TaskInput` implementation, representing some code that the debug /// tracing might be particularly useful with. -#[derive( - Copy, Clone, Debug, PartialEq, Eq, Hash, TraceRawVcs, Serialize, Deserialize, NonLocalValue, -)] +#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash, TraceRawVcs, Encode, Decode, NonLocalValue)] struct IncorrectTaskInput(ResolvedVc); impl TaskInput for IncorrectTaskInput { diff --git a/turbopack/crates/turbo-tasks-backend/tests/transient_collectible.rs b/turbopack/crates/turbo-tasks-backend/tests/transient_collectible.rs index 3de197a1e7c4e..542a843c39bc2 100644 --- a/turbopack/crates/turbo-tasks-backend/tests/transient_collectible.rs +++ b/turbopack/crates/turbo-tasks-backend/tests/transient_collectible.rs @@ -1,7 +1,7 @@ #![feature(arbitrary_self_types)] #![feature(arbitrary_self_types_pointers)] -use serde::{Deserialize, Serialize}; +use bincode::{Decode, Encode}; use turbo_tasks::{NonLocalValue, ResolvedVc, TaskInput, trace::TraceRawVcs}; use turbo_tasks_testing::{Registration, register, run_once_without_cache_check}; @@ -30,9 +30,7 @@ fn emit_incorrect_task_input_operation(value: IncorrectTaskInput) { } /// Has an intentionally incorrect `TaskInput` implementation -#[derive( - Copy, Clone, Debug, PartialEq, Eq, Hash, TraceRawVcs, Serialize, Deserialize, NonLocalValue, -)] +#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash, TraceRawVcs, Encode, Decode, NonLocalValue)] struct IncorrectTaskInput(ResolvedVc); impl TaskInput for IncorrectTaskInput { diff --git a/turbopack/crates/turbo-tasks-macros-tests/Cargo.toml b/turbopack/crates/turbo-tasks-macros-tests/Cargo.toml index 945057641bae1..ed6db780e2729 100644 --- a/turbopack/crates/turbo-tasks-macros-tests/Cargo.toml +++ b/turbopack/crates/turbo-tasks-macros-tests/Cargo.toml @@ -7,6 +7,7 @@ edition = "2024" [dev-dependencies] anyhow = { workspace = true } +bincode = { workspace = true } serde = { workspace = true } tokio = { workspace = true } trybuild = { version = "1.0.104" } diff --git a/turbopack/crates/turbo-tasks-macros-tests/tests/task_input.rs b/turbopack/crates/turbo-tasks-macros-tests/tests/task_input.rs index 7fed26c3e5517..d1f74e888a998 100644 --- a/turbopack/crates/turbo-tasks-macros-tests/tests/task_input.rs +++ b/turbopack/crates/turbo-tasks-macros-tests/tests/task_input.rs @@ -3,13 +3,13 @@ //! macro and the `#[turbo_tasks::function]` macro. #![allow(clippy::needless_return)] // tokio macro-generated code doesn't respect this -use serde::{Deserialize, Serialize}; +use bincode::{Decode, Encode}; use turbo_tasks::{Completion, ReadRef, TaskInput, Vc, trace::TraceRawVcs}; use turbo_tasks_testing::{Registration, register, run_once}; static REGISTRATION: Registration = register!(); -#[derive(Clone, TaskInput, Debug, PartialEq, Eq, Hash, Serialize, Deserialize, TraceRawVcs)] +#[derive(Clone, TaskInput, Debug, PartialEq, Eq, Hash, Encode, Decode, TraceRawVcs)] struct OneUnnamedField(u32); #[turbo_tasks::function] diff --git a/turbopack/crates/turbo-tasks/Cargo.toml b/turbopack/crates/turbo-tasks/Cargo.toml index 74f5641bd7abe..5697a6b28b575 100644 --- a/turbopack/crates/turbo-tasks/Cargo.toml +++ b/turbopack/crates/turbo-tasks/Cargo.toml @@ -56,7 +56,6 @@ turbo-tasks-hash = { workspace = true } turbo-tasks-macros = { workspace = true } turbo-tasks-malloc = { workspace = true } unsize = { workspace = true } -unty = { workspace = true } pot = "3.0.0" [dev-dependencies] diff --git a/turbopack/crates/turbo-tasks/src/backend.rs b/turbopack/crates/turbo-tasks/src/backend.rs index 0b800d2a9b8c0..a43300a7e7e4a 100644 --- a/turbopack/crates/turbo-tasks/src/backend.rs +++ b/turbopack/crates/turbo-tasks/src/backend.rs @@ -13,10 +13,14 @@ use auto_hash_map::AutoMap; use bincode::{ Decode, Encode, error::{DecodeError, EncodeError}, + impl_borrow_decode, }; use rustc_hash::FxHasher; use tracing::Span; -use turbo_bincode::{TurboBincodeDecoder, TurboBincodeEncoder}; +use turbo_bincode::{ + TurboBincodeDecode, TurboBincodeDecoder, TurboBincodeEncode, TurboBincodeEncoder, + impl_decode_for_turbo_bincode_decode, impl_encode_for_turbo_bincode_encode, +}; use turbo_rcstr::RcStr; use crate::{ @@ -77,6 +81,38 @@ impl CachedTaskType { } } +impl TurboBincodeEncode for CachedTaskType { + fn encode(&self, encoder: &mut TurboBincodeEncoder) -> Result<(), EncodeError> { + Encode::encode(®istry::get_function_id(self.native_fn), encoder)?; + + let (encode_arg_any, _) = self.native_fn.arg_meta.bincode; + Encode::encode(&self.this, encoder)?; + encode_arg_any(&*self.arg, encoder)?; + + Ok(()) + } +} + +impl TurboBincodeDecode for CachedTaskType { + fn decode(decoder: &mut TurboBincodeDecoder) -> Result { + let native_fn = registry::get_native_function(Decode::decode(decoder)?); + + let (_, decode_arg_any) = native_fn.arg_meta.bincode; + let this = Decode::decode(decoder)?; + let arg = decode_arg_any(decoder)?; + + Ok(Self { + native_fn, + this, + arg, + }) + } +} + +impl_encode_for_turbo_bincode_encode!(CachedTaskType); +impl_decode_for_turbo_bincode_decode!(CachedTaskType); +impl_borrow_decode!(CachedTaskType); + // Manual implementation is needed because of a borrow issue with `Box`: // https://github.com/rust-lang/rust/issues/31740 impl PartialEq for CachedTaskType { @@ -102,126 +138,6 @@ impl Display for CachedTaskType { } } -mod ser { - use bincode::{ - de::{Decoder, read::Reader}, - enc::Encoder, - }; - use serde::{Deserialize, Deserializer, Serialize, Serializer, ser::SerializeSeq}; - - use super::*; - - const POT_CONFIG: pot::Config = pot::Config::new().compatibility(pot::Compatibility::V4); - - struct FunctionAndArgBorrowed<'a> { - native_fn: &'static NativeFunction, - arg: &'a dyn MagicAny, - } - struct FunctionAndArgOwned { - native_fn: &'static NativeFunction, - arg: Box, - } - - impl Serialize for FunctionAndArgBorrowed<'_> { - fn serialize(&self, serializer: S) -> Result - where - S: Serializer, - { - let Self { native_fn, arg } = self; - let mut state = serializer.serialize_seq(Some(2))?; - state.serialize_element(®istry::get_function_id(native_fn))?; - let arg = *arg; - let arg = native_fn.arg_meta.as_serialize(arg); - state.serialize_element(arg)?; - state.end() - } - } - - impl<'de> Deserialize<'de> for FunctionAndArgOwned { - fn deserialize>(deserializer: D) -> Result { - struct Visitor; - impl<'de> serde::de::Visitor<'de> for Visitor { - type Value = FunctionAndArgOwned; - - fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { - write!(formatter, "a valid FunctionAndArgOwned") - } - - fn visit_seq(self, mut seq: A) -> Result - where - A: serde::de::SeqAccess<'de>, - { - let fn_id = seq - .next_element()? - .ok_or_else(|| serde::de::Error::invalid_length(0, &self))?; - let native_fn = registry::get_native_function(fn_id); - let seed = native_fn.arg_meta.deserialization_seed(); - let arg = seq - .next_element_seed(seed)? - .ok_or_else(|| serde::de::Error::invalid_length(1, &self))?; - Ok(FunctionAndArgOwned { native_fn, arg }) - } - } - deserializer.deserialize_seq(Visitor) - } - } - - // HACK: We don't yet require `TaskInput: Encode + Decode`, so use a pot serializer for the - // function arguments, and bincode for everything else. - impl Encode for CachedTaskType { - fn encode(&self, encoder: &mut E) -> Result<(), EncodeError> { - struct BincodeWriterWrapper(W); - impl std::io::Write for BincodeWriterWrapper { - fn write(&mut self, buf: &[u8]) -> std::io::Result { - self.write_all(buf)?; - Ok(buf.len()) - } - fn write_all(&mut self, buf: &[u8]) -> std::io::Result<()> { - self.0.write(buf).map_err(std::io::Error::other) - } - fn flush(&mut self) -> std::io::Result<()> { - Ok(()) - } - } - let function_and_arg = FunctionAndArgBorrowed { - native_fn: self.native_fn, - arg: &*self.arg, - }; - POT_CONFIG - .serialize_into( - &function_and_arg, - &mut BincodeWriterWrapper(encoder.writer()), - ) - .map_err(|e| EncodeError::OtherString(e.to_string()))?; - Encode::encode(&self.this, encoder) - } - } - - impl Decode for CachedTaskType { - fn decode>(decoder: &mut D) -> Result { - struct BincodeReaderWrapper(R); - impl std::io::Read for BincodeReaderWrapper { - fn read(&mut self, buf: &mut [u8]) -> std::io::Result { - self.read_exact(buf)?; - Ok(buf.len()) - } - fn read_exact(&mut self, buf: &mut [u8]) -> std::io::Result<()> { - self.0.read(buf).map_err(std::io::Error::other) - } - } - let FunctionAndArgOwned { native_fn, arg } = POT_CONFIG - .deserialize_from(BincodeReaderWrapper(decoder.reader())) - .map_err(|e| DecodeError::OtherString(e.to_string()))?; - let this: Option = Decode::decode(decoder)?; - Ok(CachedTaskType { - native_fn, - this, - arg, - }) - } - } -} - pub struct TaskExecutionSpec<'a> { pub future: Pin> + Send + 'a>>, pub span: Span, diff --git a/turbopack/crates/turbo-tasks/src/lib.rs b/turbopack/crates/turbo-tasks/src/lib.rs index 6b5276be95e8c..62fc4f773e25c 100644 --- a/turbopack/crates/turbo-tasks/src/lib.rs +++ b/turbopack/crates/turbo-tasks/src/lib.rs @@ -91,46 +91,51 @@ use std::hash::BuildHasherDefault; pub use anyhow::{Error, Result}; use auto_hash_map::AutoSet; -pub use capture_future::TurboTasksPanic; -pub use collectibles::CollectiblesSource; -pub use completion::{Completion, Completions}; -pub use display::ValueToString; -pub use effect::{ApplyEffectsContext, Effects, apply_effects, effect, get_effects}; -pub use id::{ExecutionId, LocalTaskId, TRANSIENT_TASK_BIT, TaskId, TraitTypeId, ValueTypeId}; -pub use invalidation::{ - InvalidationReason, InvalidationReasonKind, InvalidationReasonSet, Invalidator, get_invalidator, -}; -pub use join_iter_ext::{JoinIterExt, TryFlatJoinIterExt, TryJoinIterExt}; -pub use key_value_pair::KeyValuePair; -pub use magic_any::MagicAny; -pub use manager::{ - CurrentCellRef, ReadConsistency, ReadTracking, TaskPersistence, TurboTasks, TurboTasksApi, - TurboTasksBackendApi, TurboTasksCallApi, Unused, UpdateInfo, dynamic_call, emit, mark_finished, - mark_root, mark_session_dependent, mark_stateful, prevent_gc, run, run_once, - run_once_with_reason, trait_call, turbo_tasks, turbo_tasks_scope, -}; -pub use output::OutputContent; -pub use raw_vc::{CellId, RawVc, ReadRawVcFuture, ResolveTypeError}; -pub use read_options::{ReadCellOptions, ReadOutputOptions}; -pub use read_ref::ReadRef; use rustc_hash::FxHasher; -pub use serialization_invalidation::SerializationInvalidator; pub use shrink_to_fit::ShrinkToFit; -pub use spawn::{ - JoinHandle, block_for_future, block_in_place, spawn, spawn_blocking, spawn_thread, -}; -pub use state::{State, TransientState}; -pub use task::{SharedReference, TypedSharedReference, task_input::TaskInput}; -pub use task_execution_reason::TaskExecutionReason; -pub use trait_ref::{IntoTraitRef, TraitRef}; pub use turbo_tasks_macros::{TaskInput, function, value_impl}; -pub use value::{TransientInstance, TransientValue}; -pub use value_type::{TraitMethod, TraitType, ValueType}; -pub use vc::{ - Dynamic, NonLocalValue, OperationValue, OperationVc, OptionVcExt, ReadVcFuture, ResolvedVc, - Upcast, UpcastStrict, ValueDefault, Vc, VcCast, VcCellCompareMode, VcCellNewMode, - VcDefaultRead, VcRead, VcTransparentRead, VcValueTrait, VcValueTraitCast, VcValueType, - VcValueTypeCast, + +pub use crate::{ + capture_future::TurboTasksPanic, + collectibles::CollectiblesSource, + completion::{Completion, Completions}, + display::ValueToString, + effect::{ApplyEffectsContext, Effects, apply_effects, effect, get_effects}, + id::{ExecutionId, LocalTaskId, TRANSIENT_TASK_BIT, TaskId, TraitTypeId, ValueTypeId}, + invalidation::{ + InvalidationReason, InvalidationReasonKind, InvalidationReasonSet, Invalidator, + get_invalidator, + }, + join_iter_ext::{JoinIterExt, TryFlatJoinIterExt, TryJoinIterExt}, + key_value_pair::KeyValuePair, + magic_any::MagicAny, + manager::{ + CurrentCellRef, ReadConsistency, ReadTracking, TaskPersistence, TurboTasks, TurboTasksApi, + TurboTasksBackendApi, TurboTasksCallApi, Unused, UpdateInfo, dynamic_call, emit, + mark_finished, mark_root, mark_session_dependent, mark_stateful, prevent_gc, run, run_once, + run_once_with_reason, trait_call, turbo_tasks, turbo_tasks_scope, + }, + output::OutputContent, + raw_vc::{CellId, RawVc, ReadRawVcFuture, ResolveTypeError}, + read_options::{ReadCellOptions, ReadOutputOptions}, + read_ref::ReadRef, + serialization_invalidation::SerializationInvalidator, + spawn::{JoinHandle, block_for_future, block_in_place, spawn, spawn_blocking, spawn_thread}, + state::{State, TransientState}, + task::{ + SharedReference, TypedSharedReference, + task_input::{EitherTaskInput, TaskInput}, + }, + task_execution_reason::TaskExecutionReason, + trait_ref::{IntoTraitRef, TraitRef}, + value::{TransientInstance, TransientValue}, + value_type::{TraitMethod, TraitType, ValueType}, + vc::{ + Dynamic, NonLocalValue, OperationValue, OperationVc, OptionVcExt, ReadVcFuture, ResolvedVc, + Upcast, UpcastStrict, ValueDefault, Vc, VcCast, VcCellCompareMode, VcCellNewMode, + VcDefaultRead, VcRead, VcTransparentRead, VcValueTrait, VcValueTraitCast, VcValueType, + VcValueTypeCast, + }, }; pub type SliceMap = Box<[(K, V)]>; diff --git a/turbopack/crates/turbo-tasks/src/magic_any.rs b/turbopack/crates/turbo-tasks/src/magic_any.rs index 316fd1201d201..95e54a04c8fe6 100644 --- a/turbopack/crates/turbo-tasks/src/magic_any.rs +++ b/turbopack/crates/turbo-tasks/src/magic_any.rs @@ -1,6 +1,9 @@ -use std::{any::Any, fmt::Debug, hash::Hash}; +use std::{ + any::{Any, type_name}, + fmt::Debug, + hash::Hash, +}; -use serde::{Deserialize, Serialize, de::DeserializeSeed}; use turbo_dyn_eq_hash::{ DynEq, DynHash, impl_eq_for_dyn, impl_hash_for_dyn, impl_partial_eq_for_dyn, }; @@ -26,85 +29,12 @@ impl_partial_eq_for_dyn!(dyn MagicAny); impl_eq_for_dyn!(dyn MagicAny); impl_hash_for_dyn!(dyn MagicAny); -impl dyn MagicAny { - pub fn as_serialize( - &self, - ) -> &dyn erased_serde::Serialize { - if let Some(r) = (self as &dyn Any).downcast_ref::() { - r - } else { - #[cfg(debug_assertions)] - panic!( - "MagicAny::as_serializable broken: got {} but expected {}", - self.magic_type_name(), - std::any::type_name::(), - ); - #[cfg(not(debug_assertions))] - panic!("MagicAny::as_serializable bug"); - } - } -} - -type MagicAnySerializeFunctor = fn(&dyn MagicAny) -> &dyn erased_serde::Serialize; - -#[derive(Clone, Copy)] -pub struct MagicAnySerializeSeed { - functor: MagicAnySerializeFunctor, -} - -impl MagicAnySerializeSeed { - pub fn new() -> Self { - fn serialize( - value: &dyn MagicAny, - ) -> &dyn erased_serde::Serialize { - value.as_serialize::() - } - Self { - functor: serialize::, - } - } - - pub fn as_serialize<'a>(&self, value: &'a dyn MagicAny) -> &'a dyn erased_serde::Serialize { - (self.functor)(value) - } -} - -type MagicAnyDeserializeSeedFunctor = - fn(&mut dyn erased_serde::Deserializer<'_>) -> Result, erased_serde::Error>; - -#[derive(Clone, Copy)] -pub struct MagicAnyDeserializeSeed { - functor: MagicAnyDeserializeSeedFunctor, -} - -impl MagicAnyDeserializeSeed { - pub fn new() -> Self - where - T: for<'de> Deserialize<'de> + Debug + Eq + Hash + Send + Sync + TraceRawVcs + 'static, - { - fn deserialize( - deserializer: &mut dyn erased_serde::Deserializer<'_>, - ) -> Result, erased_serde::Error> - where - T: for<'de> Deserialize<'de> + Debug + Eq + Hash + Send + Sync + TraceRawVcs + 'static, - { - let value: T = erased_serde::deserialize(deserializer)?; - Ok(Box::new(value)) - } - Self { - functor: deserialize::, - } - } -} - -impl<'de> DeserializeSeed<'de> for MagicAnyDeserializeSeed { - type Value = Box; - - fn deserialize(self, deserializer: D) -> Result - where - D: serde::Deserializer<'de>, - { - let mut deserializer = ::erase(deserializer); - (self.functor)(&mut deserializer).map_err(serde::de::Error::custom) +pub fn any_as_encode(this: &dyn Any) -> &T { + if let Some(enc) = this.downcast_ref::() { + return enc; } + unreachable!( + "any_as_encode::<{}> called with invalid type", + type_name::() + ); } diff --git a/turbopack/crates/turbo-tasks/src/native_function.rs b/turbopack/crates/turbo-tasks/src/native_function.rs index 0c06ea985be71..413b7315d0d69 100644 --- a/turbopack/crates/turbo-tasks/src/native_function.rs +++ b/turbopack/crates/turbo-tasks/src/native_function.rs @@ -1,14 +1,15 @@ use std::{any::Any, fmt::Debug, hash::Hash, pin::Pin}; use anyhow::Result; +use bincode::{Decode, Encode}; use futures::Future; use once_cell::sync::Lazy; -use serde::{Deserialize, Serialize}; use tracing::Span; +use turbo_bincode::{AnyDecodeFn, AnyEncodeFn}; use crate::{ RawVc, TaskExecutionReason, TaskInput, TaskPersistence, - magic_any::{MagicAny, MagicAnyDeserializeSeed, MagicAnySerializeSeed}, + magic_any::{MagicAny, any_as_encode}, task::{ IntoTaskFn, TaskFn, function::{IntoTaskFnWithThis, NativeTaskFuture}, @@ -24,8 +25,8 @@ type FilterOwnedArgsFunctor = for<'a> fn(Box) -> Box type FilterAndResolveFunctor = ResolveFunctor; pub struct ArgMeta { - serializer: MagicAnySerializeSeed, - deserializer: MagicAnyDeserializeSeed, + // TODO: This should be an `Option` with `None` for transient tasks. We can skip some codegen. + pub bincode: (AnyEncodeFn, AnyDecodeFn>), is_resolved: IsResolvedFunctor, resolve: ResolveFunctor, /// Used for trait methods, filters out unused arguments. @@ -43,7 +44,7 @@ pub struct ArgMeta { impl ArgMeta { pub fn new() -> Self where - T: TaskInput + Serialize + for<'de> Deserialize<'de> + 'static, + T: TaskInput + Encode + Decode<()> + 'static, { fn noop_filter_args(args: Box) -> Box { args @@ -56,11 +57,19 @@ impl ArgMeta { filter_and_resolve: FilterAndResolveFunctor, ) -> Self where - T: TaskInput + Serialize + for<'de> Deserialize<'de> + 'static, + T: TaskInput + Encode + Decode<()> + 'static, { Self { - serializer: MagicAnySerializeSeed::new::(), - deserializer: MagicAnyDeserializeSeed::new::(), + bincode: ( + |this, enc| { + T::encode(any_as_encode::(this), enc)?; + Ok(()) + }, + |dec| { + let val = T::decode(dec)?; + Ok(Box::new(val)) + }, + ), is_resolved: |value| downcast_args_ref::(value).is_resolved(), resolve: resolve_functor_impl::, filter_owned, @@ -68,14 +77,6 @@ impl ArgMeta { } } - pub fn deserialization_seed(&self) -> MagicAnyDeserializeSeed { - self.deserializer - } - - pub fn as_serialize<'a>(&self, value: &'a dyn MagicAny) -> &'a dyn erased_serde::Serialize { - self.serializer.as_serialize(value) - } - pub fn is_resolved(&self, value: &dyn MagicAny) -> bool { (self.is_resolved)(value) } @@ -174,7 +175,7 @@ impl NativeFunction { implementation: impl IntoTaskFn, ) -> Self where - Inputs: TaskInput + Serialize + for<'de> Deserialize<'de> + 'static, + Inputs: TaskInput + Encode + Decode<()> + 'static, { Self { name, @@ -191,7 +192,7 @@ impl NativeFunction { implementation: I, ) -> Self where - Inputs: TaskInput + Serialize + for<'de> Deserialize<'de> + 'static, + Inputs: TaskInput + Encode + Decode<()> + 'static, I: IntoTaskFn, { Self { @@ -214,7 +215,7 @@ impl NativeFunction { ) -> Self where This: Sync + Send + 'static, - Inputs: TaskInput + Serialize + for<'de> Deserialize<'de> + 'static, + Inputs: TaskInput + Encode + Decode<()> + 'static, I: IntoTaskFnWithThis, { Self { diff --git a/turbopack/crates/turbo-tasks/src/task/shared_reference.rs b/turbopack/crates/turbo-tasks/src/task/shared_reference.rs index 76575845246a5..0973989632efd 100644 --- a/turbopack/crates/turbo-tasks/src/task/shared_reference.rs +++ b/turbopack/crates/turbo-tasks/src/task/shared_reference.rs @@ -8,13 +8,12 @@ use std::{ use anyhow::Result; use bincode::{ Decode, Encode, - de::Decoder, - enc::Encoder, error::{DecodeError, EncodeError}, impl_borrow_decode, }; use turbo_bincode::{ - TurboBincodeDecoder, TurboBincodeEncoder, turbo_bincode_decode, turbo_bincode_encode, + TurboBincodeDecode, TurboBincodeDecoder, TurboBincodeEncode, TurboBincodeEncoder, + impl_decode_for_turbo_bincode_decode, impl_encode_for_turbo_bincode_encode, }; use unsize::CoerceUnsize; @@ -64,99 +63,44 @@ impl TypedSharedReference { pub fn into_untyped(self) -> SharedReference { self.reference } +} - fn encode(&self, enc: &mut TurboBincodeEncoder) -> Result<(), EncodeError> { +impl TurboBincodeEncode for TypedSharedReference { + fn encode(&self, encoder: &mut TurboBincodeEncoder) -> Result<(), EncodeError> { let Self { type_id, reference } = self; let value_type = registry::get_value_type(*type_id); if let Some(bincode) = value_type.bincode { - type_id.encode(enc)?; - bincode.0(&*reference.0, enc)?; + type_id.encode(encoder)?; + bincode.0(&*reference.0, encoder)?; Ok(()) } else { Err(EncodeError::OtherString(format!( - "{:?} is not serializable", + "{} is not encodable", value_type.global_name ))) } } +} - fn decode(dec: &mut TurboBincodeDecoder) -> Result { - let type_id = ValueTypeId::decode(dec)?; +impl TurboBincodeDecode for TypedSharedReference { + fn decode(decoder: &mut TurboBincodeDecoder) -> Result { + let type_id = ValueTypeId::decode(decoder)?; let value_type = registry::get_value_type(type_id); if let Some(bincode) = value_type.bincode { - let reference = bincode.1(dec)?; + let reference = bincode.1(decoder)?; Ok(Self { type_id, reference }) } else { #[cold] - fn not_deserializable(value_type: &ValueType) -> DecodeError { - DecodeError::OtherString(format!("{value_type} is not deserializable")) - } - Err(not_deserializable(value_type)) - } - } -} - -impl Encode for TypedSharedReference { - fn encode<'a, E: Encoder>(&self, encoder: &'a mut E) -> Result<(), EncodeError> { - let maybe_turbo_encoder = if unty::type_equal::() { - // SAFETY: Transmute is safe because `&mut E` is `&mut TurboBincodeEncoder`: - // - `unty::type_equal::()` does not check lifetimes, but does - // check the type and layout, so we know those are correct. - // - The transmuted encoder cannot escape this function, and we know that the lifetime - // of `'f` is at least as long as the function. - // - Lifetimes don't change layout. This is not guaranteed, but if this assumption is - // broken, we'd have a different type id, `type_equal` would return `false` and we'd - // fall back to a slower codepath, and wouldn't violate memory safety. - // - Two mutable references have the same layout and alignment when they reference - // exactly the same type. - // - The explicit lifetime ('a) avoids creating an implitly unbounded lifetime. - Ok(unsafe { std::mem::transmute::<&'a mut E, &'a mut TurboBincodeEncoder>(encoder) }) - } else { - Err(encoder) - }; - match maybe_turbo_encoder { - Ok(turbo_encoder) => TypedSharedReference::encode(self, turbo_encoder), - Err(generic_encoder) => { - // The underlying `SharedReference` can only be serialized using - // `TurboBincodeEncoder` because the encoder function pointer cannot take type - // parameters. This is okay, because we expect any hot codepaths to use - // `TurboBincodeEncoder`. - // - // Create a `TurboBincodeEncoder` and encode this as a nested byte array. We must - // redundantly store a size here, otherwise we won't be able to determine what size - // buffer to use for `TurboBincodeReader`. - let buffer = turbo_bincode_encode(self)?; - buffer.encode(generic_encoder) - } - } - } -} - -impl Decode for TypedSharedReference { - fn decode<'a, D: Decoder>(decoder: &mut D) -> Result { - let maybe_turbo_decoder = if unty::type_equal::() { - // SAFETY: See notes on the `Encode::encode` implementation above. - Ok(unsafe { std::mem::transmute::<&mut D, &mut TurboBincodeDecoder<'a>>(decoder) }) - } else { - Err(decoder) - }; - match maybe_turbo_decoder { - Ok(turbo_decoder) => TypedSharedReference::decode(turbo_decoder), - Err(generic_decoder) => { - // The underlying `SharedReference` can only be deserialized using - // `TurboBincodeDecoder` because the decoder function pointer cannot take type - // parameters. This is okay, because we expect any hot codepaths to use - // `TurboBincodeDecoder`. - // - // Decode the nested byte array that was created during encoding, then use a - // `TurboBincodeDecoder` to decode the contents. - let buffer: Vec = Decode::decode(generic_decoder)?; - turbo_bincode_decode(&buffer) + fn not_decodable(value_type: &ValueType) -> DecodeError { + DecodeError::OtherString(format!("{} is not decodable", value_type.global_name)) } + Err(not_decodable(value_type)) } } } +impl_encode_for_turbo_bincode_encode!(TypedSharedReference); +impl_decode_for_turbo_bincode_decode!(TypedSharedReference); impl_borrow_decode!(TypedSharedReference); impl Deref for TypedSharedReference { diff --git a/turbopack/crates/turbo-tasks/src/task/task_input.rs b/turbopack/crates/turbo-tasks/src/task/task_input.rs index f981e51bacb96..1a466303d5abe 100644 --- a/turbopack/crates/turbo-tasks/src/task/task_input.rs +++ b/turbopack/crates/turbo-tasks/src/task/task_input.rs @@ -3,15 +3,24 @@ use std::{ fmt::Debug, future::Future, hash::Hash, + ops::{Deref, DerefMut}, sync::Arc, time::Duration, }; use anyhow::Result; +use bincode::{ + Decode, Encode, + de::Decoder, + enc::Encoder, + error::{DecodeError, EncodeError}, +}; use either::Either; -use serde::{Deserialize, Serialize}; use turbo_rcstr::RcStr; +// This import is necessary for derive macros to work, as their expansion refers to the crate +// name directly. +use crate as turbo_tasks; use crate::{ MagicAny, ReadRef, ResolvedVc, TaskId, TransientInstance, TransientValue, ValueTypeId, Vc, trace::TraceRawVcs, @@ -20,9 +29,14 @@ use crate::{ /// Trait to implement in order for a type to be accepted as a /// [`#[turbo_tasks::function]`][crate::function] argument. /// -/// Serialization must be deterministic and compatible with `eq` comparisons. If two `TaskInputs` -/// compare equal they must also serialize to the same bytes. -pub trait TaskInput: Send + Sync + Clone + Debug + PartialEq + Eq + Hash + TraceRawVcs { +/// Transient task inputs are required to implement [`Encode`] and [`Decode`], but are allowed to +/// panic at runtime. This requirement could be lifted in the future. +/// +/// Bincode encoding must be deterministic and compatible with [`Eq`] comparisons. If two +/// `TaskInput`s compare equal they must also encode to the same bytes. +pub trait TaskInput: + Send + Sync + Clone + Debug + PartialEq + Eq + Hash + TraceRawVcs + Encode + Decode<()> +{ fn resolve_input(&self) -> impl Future> + Send + '_ { async { Ok(self.clone()) } } @@ -205,31 +219,15 @@ where } } -impl Serialize for TransientValue -where - T: MagicAny + Clone + 'static, -{ - fn serialize(&self, _serializer: S) -> Result - where - S: serde::Serializer, - { - Err(serde::ser::Error::custom( - "cannot serialize transient task inputs", - )) +impl Encode for TransientValue { + fn encode(&self, _encoder: &mut E) -> Result<(), EncodeError> { + Err(EncodeError::Other("cannot encode transient task inputs")) } } -impl<'de, T> Deserialize<'de> for TransientValue -where - T: MagicAny + Clone + 'static, -{ - fn deserialize(_deserializer: D) -> Result - where - D: serde::Deserializer<'de>, - { - Err(serde::de::Error::custom( - "cannot deserialize transient task inputs", - )) +impl Decode for TransientValue { + fn decode>(_decoder: &mut D) -> Result { + Err(DecodeError::Other("cannot decode transient task inputs")) } } @@ -242,48 +240,15 @@ where } } -impl Serialize for TransientInstance { - fn serialize(&self, _serializer: S) -> Result - where - S: serde::Serializer, - { - Err(serde::ser::Error::custom( - "cannot serialize transient task inputs", - )) +impl Encode for TransientInstance { + fn encode(&self, _encoder: &mut E) -> Result<(), EncodeError> { + Err(EncodeError::Other("cannot encode transient task inputs")) } } -impl<'de, T> Deserialize<'de> for TransientInstance { - fn deserialize(_deserializer: D) -> Result - where - D: serde::Deserializer<'de>, - { - Err(serde::de::Error::custom( - "cannot deserialize transient task inputs", - )) - } -} - -impl TaskInput for Either -where - L: TaskInput, - R: TaskInput, -{ - fn resolve_input(&self) -> impl Future> + Send + '_ { - self.as_ref().map_either( - |l| async move { anyhow::Ok(Either::Left(l.resolve_input().await?)) }, - |r| async move { anyhow::Ok(Either::Right(r.resolve_input().await?)) }, - ) - } - - fn is_resolved(&self) -> bool { - self.as_ref() - .either(TaskInput::is_resolved, TaskInput::is_resolved) - } - - fn is_transient(&self) -> bool { - self.as_ref() - .either(TaskInput::is_transient, TaskInput::is_transient) +impl Decode for TransientInstance { + fn decode>(_decoder: &mut D) -> Result { + Err(DecodeError::Other("cannot decode transient task inputs")) } } @@ -335,6 +300,68 @@ where } } +/// A thin wrapper around [`Either`] that implements the traits required by [`TaskInput`], notably +/// [`Encode`] and [`Decode`]. +#[derive(Clone, Debug, PartialEq, Eq, Hash, TraceRawVcs)] +pub struct EitherTaskInput(pub Either); + +impl Deref for EitherTaskInput { + type Target = Either; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl DerefMut for EitherTaskInput { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.0 + } +} + +impl Encode for EitherTaskInput +where + L: Encode, + R: Encode, +{ + fn encode(&self, encoder: &mut E) -> Result<(), EncodeError> { + turbo_bincode::either::encode(self, encoder) + } +} + +impl Decode for EitherTaskInput +where + L: Decode, + R: Decode, +{ + fn decode>(decoder: &mut D) -> Result { + turbo_bincode::either::decode(decoder).map(Self) + } +} + +impl TaskInput for EitherTaskInput +where + L: TaskInput, + R: TaskInput, +{ + fn resolve_input(&self) -> impl Future> + Send + '_ { + self.as_ref().map_either( + |l| async move { anyhow::Ok(Self(Either::Left(l.resolve_input().await?))) }, + |r| async move { anyhow::Ok(Self(Either::Right(r.resolve_input().await?))) }, + ) + } + + fn is_resolved(&self) -> bool { + self.as_ref() + .either(TaskInput::is_resolved, TaskInput::is_resolved) + } + + fn is_transient(&self) -> bool { + self.as_ref() + .either(TaskInput::is_transient, TaskInput::is_transient) + } +} + macro_rules! tuple_impls { ( $( $name:ident )+ ) => { impl<$($name: TaskInput),+> TaskInput for ($($name,)+) @@ -381,9 +408,6 @@ mod tests { use turbo_tasks_macros::TaskInput; use super::*; - // This is necessary for the derive macro to work, as its expansion refers to - // the crate name directly. - use crate as turbo_tasks; fn assert_task_input(_: T) where @@ -393,9 +417,7 @@ mod tests { #[test] fn test_no_fields() -> Result<()> { - #[derive( - Clone, TaskInput, Eq, PartialEq, Hash, Debug, Serialize, Deserialize, TraceRawVcs, - )] + #[derive(Clone, TaskInput, Eq, PartialEq, Hash, Debug, Encode, Decode, TraceRawVcs)] struct NoFields; assert_task_input(NoFields); @@ -404,9 +426,7 @@ mod tests { #[test] fn test_one_unnamed_field() -> Result<()> { - #[derive( - Clone, TaskInput, Eq, PartialEq, Hash, Debug, Serialize, Deserialize, TraceRawVcs, - )] + #[derive(Clone, TaskInput, Eq, PartialEq, Hash, Debug, Encode, Decode, TraceRawVcs)] struct OneUnnamedField(u32); assert_task_input(OneUnnamedField(42)); @@ -415,9 +435,7 @@ mod tests { #[test] fn test_multiple_unnamed_fields() -> Result<()> { - #[derive( - Clone, TaskInput, Eq, PartialEq, Hash, Debug, Serialize, Deserialize, TraceRawVcs, - )] + #[derive(Clone, TaskInput, Eq, PartialEq, Hash, Debug, Encode, Decode, TraceRawVcs)] struct MultipleUnnamedFields(u32, RcStr); assert_task_input(MultipleUnnamedFields(42, rcstr!("42"))); @@ -426,9 +444,7 @@ mod tests { #[test] fn test_one_named_field() -> Result<()> { - #[derive( - Clone, TaskInput, Eq, PartialEq, Hash, Debug, Serialize, Deserialize, TraceRawVcs, - )] + #[derive(Clone, TaskInput, Eq, PartialEq, Hash, Debug, Encode, Decode, TraceRawVcs)] struct OneNamedField { named: u32, } @@ -439,9 +455,7 @@ mod tests { #[test] fn test_multiple_named_fields() -> Result<()> { - #[derive( - Clone, TaskInput, Eq, PartialEq, Hash, Debug, Serialize, Deserialize, TraceRawVcs, - )] + #[derive(Clone, TaskInput, Eq, PartialEq, Hash, Debug, Encode, Decode, TraceRawVcs)] struct MultipleNamedFields { named: u32, other: RcStr, @@ -456,9 +470,7 @@ mod tests { #[test] fn test_generic_field() -> Result<()> { - #[derive( - Clone, TaskInput, Eq, PartialEq, Hash, Debug, Serialize, Deserialize, TraceRawVcs, - )] + #[derive(Clone, TaskInput, Eq, PartialEq, Hash, Debug, Encode, Decode, TraceRawVcs)] struct GenericField(T); assert_task_input(GenericField(42)); @@ -466,7 +478,7 @@ mod tests { Ok(()) } - #[derive(Clone, TaskInput, Eq, PartialEq, Hash, Debug, Serialize, Deserialize, TraceRawVcs)] + #[derive(Clone, TaskInput, Eq, PartialEq, Hash, Debug, Encode, Decode, TraceRawVcs)] enum OneVariant { Variant, } @@ -479,9 +491,7 @@ mod tests { #[test] fn test_multiple_variants() -> Result<()> { - #[derive( - Clone, TaskInput, PartialEq, Eq, Hash, Debug, Serialize, Deserialize, TraceRawVcs, - )] + #[derive(Clone, TaskInput, PartialEq, Eq, Hash, Debug, Encode, Decode, TraceRawVcs)] enum MultipleVariants { Variant1, Variant2, @@ -491,7 +501,7 @@ mod tests { Ok(()) } - #[derive(Clone, TaskInput, Eq, PartialEq, Hash, Debug, Serialize, Deserialize, TraceRawVcs)] + #[derive(Clone, TaskInput, Eq, PartialEq, Hash, Debug, Encode, Decode, TraceRawVcs)] enum MultipleVariantsAndHeterogeneousFields { Variant1, Variant2(u32), @@ -511,9 +521,7 @@ mod tests { #[test] fn test_nested_variants() -> Result<()> { - #[derive( - Clone, TaskInput, Eq, PartialEq, Hash, Debug, Serialize, Deserialize, TraceRawVcs, - )] + #[derive(Clone, TaskInput, Eq, PartialEq, Hash, Debug, Encode, Decode, TraceRawVcs)] enum NestedVariants { Variant1, Variant2(MultipleVariantsAndHeterogeneousFields), diff --git a/turbopack/crates/turbo-tasks/src/value_type.rs b/turbopack/crates/turbo-tasks/src/value_type.rs index 38e560bd74721..66fc73c1f4875 100644 --- a/turbopack/crates/turbo-tasks/src/value_type.rs +++ b/turbopack/crates/turbo-tasks/src/value_type.rs @@ -1,27 +1,21 @@ use std::{ - any::{Any, type_name}, fmt::{self, Debug, Display, Formatter}, hash::Hash, }; use auto_hash_map::{AutoMap, AutoSet}; -use bincode::{ - Decode, Encode, - error::{DecodeError, EncodeError}, -}; +use bincode::{Decode, Encode}; use tracing::Span; -use turbo_bincode::{TurboBincodeDecoder, TurboBincodeEncoder}; +use turbo_bincode::{AnyDecodeFn, AnyEncodeFn}; use crate::{ - RawVc, SharedReference, VcValueType, id::TraitTypeId, macro_helpers::NativeFunction, registry, - task::shared_reference::TypedSharedReference, vc::VcCellMode, + RawVc, SharedReference, VcValueType, id::TraitTypeId, macro_helpers::NativeFunction, + magic_any::any_as_encode, registry, task::shared_reference::TypedSharedReference, + vc::VcCellMode, }; type RawCellFactoryFn = fn(TypedSharedReference) -> RawVc; -type AnyEncodeFn = fn(&dyn Any, &mut TurboBincodeEncoder<'_>) -> Result<(), EncodeError>; -type AnyDecodeFn = fn(&mut TurboBincodeDecoder<'_>) -> Result; - // TODO this type need some refactoring when multiple languages are added to // turbo-task In this case a trait_method might be of a different function type. // It probably need to be a Vc. @@ -42,7 +36,7 @@ pub struct ValueType { trait_methods: AutoMap<&'static TraitMethod, &'static NativeFunction>, /// Functions to convert to write the type to a buffer or read it from a buffer. - pub bincode: Option<(AnyEncodeFn, AnyDecodeFn)>, + pub bincode: Option<(AnyEncodeFn, AnyDecodeFn)>, /// An implementation of /// [`VcCellMode::raw_cell`][crate::vc::VcCellMode::raw_cell]. @@ -91,16 +85,6 @@ impl Display for ValueType { } } -pub fn any_as_encode(this: &dyn Any) -> &T { - if let Some(enc) = this.downcast_ref::() { - return enc; - } - unreachable!( - "any_as_encode::<{}> called with invalid type", - type_name::() - ); -} - pub trait ManualEncodeWrapper: Encode { type Value; @@ -171,7 +155,7 @@ impl ValueType { // Helper for other constructor functions fn new_inner( global_name: &'static str, - bincode: Option<(AnyEncodeFn, AnyDecodeFn)>, + bincode: Option<(AnyEncodeFn, AnyDecodeFn)>, ) -> Self { Self { name: std::any::type_name::(), diff --git a/turbopack/crates/turbopack-cli/Cargo.toml b/turbopack/crates/turbopack-cli/Cargo.toml index bc37b050ec5b5..d86ef5f215527 100644 --- a/turbopack/crates/turbopack-cli/Cargo.toml +++ b/turbopack/crates/turbopack-cli/Cargo.toml @@ -38,6 +38,7 @@ workspace = true [dependencies] anyhow = { workspace = true } +bincode = { workspace = true } clap = { workspace = true, features = ["derive", "env"] } console-subscriber = { workspace = true, optional = true } dunce = { workspace = true } diff --git a/turbopack/crates/turbopack-cli/src/arguments.rs b/turbopack/crates/turbopack-cli/src/arguments.rs index 41e5289b9b66d..b3a2228192fdc 100644 --- a/turbopack/crates/turbopack-cli/src/arguments.rs +++ b/turbopack/crates/turbopack-cli/src/arguments.rs @@ -5,6 +5,7 @@ use std::{ }; use anyhow::anyhow; +use bincode::{Decode, Encode}; use clap::{Args, Parser, ValueEnum}; use serde::{Deserialize, Serialize}; use turbo_tasks::{NonLocalValue, TaskInput, trace::TraceRawVcs}; @@ -48,6 +49,8 @@ impl Arguments { TaskInput, NonLocalValue, TraceRawVcs, + Encode, + Decode, )] pub enum Target { Browser, diff --git a/turbopack/crates/turbopack-cli/src/util.rs b/turbopack/crates/turbopack-cli/src/util.rs index 5d533b7b492c3..3219d566c6635 100644 --- a/turbopack/crates/turbopack-cli/src/util.rs +++ b/turbopack/crates/turbopack-cli/src/util.rs @@ -1,6 +1,7 @@ use std::{env::current_dir, path::PathBuf}; use anyhow::{Context, Result}; +use bincode::{Decode, Encode}; use dunce::canonicalize; use serde::{Deserialize, Serialize}; use turbo_rcstr::{RcStr, rcstr}; @@ -8,7 +9,18 @@ use turbo_tasks::{NonLocalValue, TaskInput, Vc, trace::TraceRawVcs}; use turbo_tasks_fs::{DiskFileSystem, FileSystem}; #[derive( - Clone, Debug, TaskInput, Hash, PartialEq, Eq, NonLocalValue, Serialize, Deserialize, TraceRawVcs, + Clone, + Debug, + TaskInput, + Hash, + PartialEq, + Eq, + NonLocalValue, + Serialize, + Deserialize, + TraceRawVcs, + Encode, + Decode, )] pub enum EntryRequest { Relative(RcStr), diff --git a/turbopack/crates/turbopack-core/src/module_graph/style_groups.rs b/turbopack/crates/turbopack-core/src/module_graph/style_groups.rs index f9c750723ad08..ce5f068fd5632 100644 --- a/turbopack/crates/turbopack-core/src/module_graph/style_groups.rs +++ b/turbopack/crates/turbopack-core/src/module_graph/style_groups.rs @@ -1,6 +1,7 @@ use std::cmp::Reverse; use anyhow::Result; +use bincode::{Decode, Encode}; use indexmap::map::Entry; use rustc_hash::{FxHashMap, FxHashSet}; use serde::{Deserialize, Serialize}; @@ -23,7 +24,18 @@ use crate::{ }; #[derive( - TaskInput, Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize, NonLocalValue, TraceRawVcs, + TaskInput, + Debug, + Clone, + PartialEq, + Eq, + Hash, + Serialize, + Deserialize, + NonLocalValue, + TraceRawVcs, + Encode, + Decode, )] pub struct StyleGroupsConfig { pub max_chunk_size: usize, diff --git a/turbopack/crates/turbopack-dev-server/src/source/headers.rs b/turbopack/crates/turbopack-dev-server/src/source/headers.rs index 769c48b650134..311cfcebdf10a 100644 --- a/turbopack/crates/turbopack-dev-server/src/source/headers.rs +++ b/turbopack/crates/turbopack-dev-server/src/source/headers.rs @@ -1,5 +1,6 @@ use std::{collections::BTreeMap, hash::Hash, mem::replace, ops::DerefMut}; +use bincode::{Decode, Encode}; use serde::{Deserialize, Serialize}; use turbo_tasks::{NonLocalValue, TaskInput, trace::TraceRawVcs}; @@ -16,6 +17,8 @@ use turbo_tasks::{NonLocalValue, TaskInput, trace::TraceRawVcs}; Deserialize, NonLocalValue, TaskInput, + Encode, + Decode, )] #[serde(transparent)] pub struct Headers(BTreeMap); @@ -23,7 +26,18 @@ pub struct Headers(BTreeMap); /// The value of an http header. HTTP headers might contain non-utf-8 bytes. An /// header might also occur multiple times. #[derive( - Clone, Debug, PartialEq, Eq, Hash, TraceRawVcs, Serialize, Deserialize, NonLocalValue, TaskInput, + Clone, + Debug, + PartialEq, + Eq, + Hash, + TraceRawVcs, + Serialize, + Deserialize, + NonLocalValue, + TaskInput, + Encode, + Decode, )] #[serde(untagged)] pub enum HeaderValue { diff --git a/turbopack/crates/turbopack-dev-server/src/source/mod.rs b/turbopack/crates/turbopack-dev-server/src/source/mod.rs index fb9aef416706b..24eb3d43086ed 100644 --- a/turbopack/crates/turbopack-dev-server/src/source/mod.rs +++ b/turbopack/crates/turbopack-dev-server/src/source/mod.rs @@ -28,7 +28,7 @@ use turbo_tasks_fs::FileSystemPath; use turbo_tasks_hash::{DeterministicHash, DeterministicHasher, Xxh3Hash64Hasher}; use turbopack_core::version::{Version, VersionedContent}; -use self::{ +use crate::source::{ headers::Headers, issue_context::IssueFilePathContentSource, query::Query, route_tree::RouteTree, }; @@ -192,6 +192,8 @@ impl HeaderList { Hash, Default, TaskInput, + Encode, + Decode, )] pub struct ContentSourceData { /// HTTP method, if requested. diff --git a/turbopack/crates/turbopack-dev-server/src/source/query.rs b/turbopack/crates/turbopack-dev-server/src/source/query.rs index 1d243758cd4e5..1733d3a2c6543 100644 --- a/turbopack/crates/turbopack-dev-server/src/source/query.rs +++ b/turbopack/crates/turbopack-dev-server/src/source/query.rs @@ -1,13 +1,25 @@ use std::{collections::BTreeMap, hash::Hash, ops::DerefMut}; +use bincode::{Decode, Encode}; use serde::{Deserialize, Serialize}; use turbo_tasks::{NonLocalValue, TaskInput, trace::TraceRawVcs}; -use super::ContentSourceDataFilter; +use crate::source::ContentSourceDataFilter; /// A parsed query string from a http request #[derive( - Clone, Debug, PartialEq, Eq, Default, Hash, TraceRawVcs, Serialize, Deserialize, NonLocalValue, + Clone, + Debug, + PartialEq, + Eq, + Default, + Hash, + TraceRawVcs, + Serialize, + Deserialize, + NonLocalValue, + Encode, + Decode, )] #[serde(transparent)] pub struct Query(BTreeMap); @@ -44,7 +56,19 @@ impl DerefMut for Query { } } -#[derive(Clone, Debug, PartialEq, Eq, Hash, TraceRawVcs, Serialize, Deserialize, NonLocalValue)] +#[derive( + Clone, + Debug, + PartialEq, + Eq, + Hash, + TraceRawVcs, + Serialize, + Deserialize, + NonLocalValue, + Encode, + Decode, +)] #[serde(untagged)] pub enum QueryValue { /// Simple string value, might be an empty string when there is no value diff --git a/turbopack/crates/turbopack-dev-server/src/source/route_tree.rs b/turbopack/crates/turbopack-dev-server/src/source/route_tree.rs index 70bc99d530d70..f2821ee77a635 100644 --- a/turbopack/crates/turbopack-dev-server/src/source/route_tree.rs +++ b/turbopack/crates/turbopack-dev-server/src/source/route_tree.rs @@ -14,7 +14,18 @@ use crate::source::{GetContentSourceContent, GetContentSourceContents}; /// The type of the route. This will decide about the remaining segments of the /// route after the base. #[derive( - TaskInput, Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize, TraceRawVcs, NonLocalValue, + TaskInput, + Clone, + Debug, + PartialEq, + Eq, + Hash, + Serialize, + Deserialize, + TraceRawVcs, + NonLocalValue, + Encode, + Decode, )] pub enum RouteType { Exact, diff --git a/turbopack/crates/turbopack-ecmascript/src/references/pattern_mapping.rs b/turbopack/crates/turbopack-ecmascript/src/references/pattern_mapping.rs index 28059c1741a78..d6204e64a4bd7 100644 --- a/turbopack/crates/turbopack-ecmascript/src/references/pattern_mapping.rs +++ b/turbopack/crates/turbopack-ecmascript/src/references/pattern_mapping.rs @@ -109,6 +109,8 @@ pub(crate) enum PatternMapping { TraceRawVcs, TaskInput, NonLocalValue, + Encode, + Decode, )] pub(crate) enum ResolveType { AsyncChunkLoader, diff --git a/turbopack/crates/turbopack-node/src/evaluate.rs b/turbopack/crates/turbopack-node/src/evaluate.rs index e7141b73239b8..cc3bc140a8cd5 100644 --- a/turbopack/crates/turbopack-node/src/evaluate.rs +++ b/turbopack/crates/turbopack-node/src/evaluate.rs @@ -1,6 +1,7 @@ use std::{borrow::Cow, iter, sync::Arc, thread::available_parallelism, time::Duration}; use anyhow::{Result, bail}; +use bincode::{Decode, Encode}; use futures_retry::{FutureRetry, RetryPolicy}; use serde::{Deserialize, Serialize, de::DeserializeOwned}; use serde_json::Value as JsonValue; @@ -143,6 +144,8 @@ async fn emit_evaluate_pool_assets_with_effects_operation( TaskInput, NonLocalValue, TraceRawVcs, + Encode, + Decode, )] pub enum EnvVarTracking { WholeEnvTracked, @@ -509,7 +512,6 @@ async fn pull_operation( } } -#[derive(Clone, PartialEq, Eq, Hash, TaskInput, Debug, Serialize, Deserialize, TraceRawVcs)] struct BasicEvaluateContext { entries: ResolvedVc, cwd: FileSystemPath, diff --git a/turbopack/crates/turbopack-node/src/transforms/webpack.rs b/turbopack/crates/turbopack-node/src/transforms/webpack.rs index d1576e0fd8160..34047af1d32ef 100644 --- a/turbopack/crates/turbopack-node/src/transforms/webpack.rs +++ b/turbopack/crates/turbopack-node/src/transforms/webpack.rs @@ -395,7 +395,19 @@ pub enum InfoMessage { }, } -#[derive(Debug, Clone, TaskInput, Hash, PartialEq, Eq, Serialize, Deserialize, TraceRawVcs)] +#[derive( + Debug, + Clone, + TaskInput, + Hash, + PartialEq, + Eq, + Serialize, + Deserialize, + TraceRawVcs, + Encode, + Decode, +)] #[serde(rename_all = "camelCase")] pub struct WebpackResolveOptions { alias_fields: Option>, @@ -430,7 +442,19 @@ pub enum ResponseMessage { TrackFileRead {}, } -#[derive(Clone, PartialEq, Eq, Hash, TaskInput, Serialize, Deserialize, Debug, TraceRawVcs)] +#[derive( + Clone, + PartialEq, + Eq, + Hash, + TaskInput, + Serialize, + Deserialize, + Debug, + TraceRawVcs, + Encode, + Decode, +)] pub struct WebpackLoaderContext { pub entries: ResolvedVc, pub cwd: FileSystemPath,