From 456972e751ce8a9be338eb519b3de68e855f7ea6 Mon Sep 17 00:00:00 2001 From: evren Date: Fri, 22 Nov 2024 14:50:28 +0300 Subject: [PATCH 01/24] Split block --- README.md | 28 +++ lib/protoflow-blocks/doc/flow/split.mmd | 15 ++ lib/protoflow-blocks/doc/flow/split.seq.mmd | 29 +++ lib/protoflow-blocks/src/block_tag.rs | 4 + lib/protoflow-blocks/src/blocks/flow.rs | 62 ++++++- lib/protoflow-blocks/src/blocks/flow/split.rs | 170 ++++++++++++++++++ lib/protoflow-blocks/src/lib.rs | 1 + lib/protoflow-blocks/src/system.rs | 10 +- 8 files changed, 308 insertions(+), 11 deletions(-) create mode 100644 lib/protoflow-blocks/doc/flow/split.mmd create mode 100644 lib/protoflow-blocks/doc/flow/split.seq.mmd create mode 100644 lib/protoflow-blocks/src/blocks/flow/split.rs diff --git a/README.md b/README.md index a6222644..7e71d511 100644 --- a/README.md +++ b/README.md @@ -133,6 +133,7 @@ The built-in blocks provided by Protoflow are listed below: | [`ReadFile`] | Reads bytes from the contents of a file. | | [`ReadSocket`] | Reads bytes from a TCP socket. | | [`ReadStdin`] | Reads bytes from standard input (aka stdin). | +| [`Split`] | Divides a single input message stream into multiple output streams using a round-robin approach. | | [`SplitString`] | Splits the received input message, with an optional delimiter string parameter. | | [`WriteFile`] | Writes or appends bytes to the contents of a file. | | [`WriteSocket`] | Writes bytes to a TCP socket | @@ -618,6 +619,32 @@ block-beta protoflow execute ReadStdin < input.txt ``` +#### [`Split`] + +Divides a single input message stream into multiple output streams using a round-robin approach. + +```mermaid +block-beta + columns 7 + space:5 Sink1 space:1 + space:1 Source space:1 Split space:3 + space:5 Sink2 space:1 + Source-- "input" -->Split + Split-- "output_1" -->Sink1 + Split-- "output_2" -->Sink2 + + classDef block height:48px,padding:8px; + classDef hidden visibility:none; + class Split block + class Source hidden + class Sink1 hidden + class Sink2 hidden +``` + +```bash +protoflow execute Split +``` + #### [`SplitString`] A block that splits the received input message, with an optional delimiter string parameter @@ -815,6 +842,7 @@ To add a new block type implementation, make sure to examine and amend: [`ReadFile`]: https://docs.rs/protoflow-blocks/latest/protoflow_blocks/struct.ReadFile.html [`ReadSocket`]: https://docs.rs/protoflow-blocks/latest/protoflow_blocks/struct.ReadSocket.html [`ReadStdin`]: https://docs.rs/protoflow-blocks/latest/protoflow_blocks/struct.ReadStdin.html +[`Split`]: https://docs.rs/protoflow-blocks/latest/protoflow_blocks/struct.Split.html [`SplitString`]: https://docs.rs/protoflow-blocks/latest/protoflow_blocks/struct.SplitString.html [`WriteFile`]: https://docs.rs/protoflow-blocks/latest/protoflow_blocks/struct.WriteFile.html [`WriteSocket`]: https://docs.rs/protoflow-blocks/latest/protoflow_blocks/struct.WriteSocket.html diff --git a/lib/protoflow-blocks/doc/flow/split.mmd b/lib/protoflow-blocks/doc/flow/split.mmd new file mode 100644 index 00000000..55bb0aad --- /dev/null +++ b/lib/protoflow-blocks/doc/flow/split.mmd @@ -0,0 +1,15 @@ +block-beta + columns 7 + space:5 Sink1 space:1 + space:1 Source space:1 Split space:3 + space:5 Sink2 space:1 + Source-- "input" -->Split + Split-- "output_1" -->Sink1 + Split-- "output_2" -->Sink2 + + classDef block height:48px,padding:8px; + classDef hidden visibility:none; + class Split block + class Source hidden + class Sink1 hidden + class Sink2 hidden diff --git a/lib/protoflow-blocks/doc/flow/split.seq.mmd b/lib/protoflow-blocks/doc/flow/split.seq.mmd new file mode 100644 index 00000000..3f61f282 --- /dev/null +++ b/lib/protoflow-blocks/doc/flow/split.seq.mmd @@ -0,0 +1,29 @@ +sequenceDiagram + autonumber + participant BlockA as Another block + participant Split.input as Split.input port + participant Split as Split block + participant Split.output_1 as Split.output_1 port + participant BlockB as Another block + participant Split.output_2 as Split.output_2 port + participant BlockC as Another block + + BlockA-->>Split: Connect + Split-->>BlockB: Connect + Split-->>BlockC: Connect + + loop Split process + BlockA->>Split: Message + Split->>Split: Decide the next output port + Split->>BlockB: Message + BlockA->>Split: Message + Split->>Split: Decide the next output port + Split->>BlockC: Message + end + + BlockA-->>Split: Disconnect + Split-->>Split.input: Close + Split-->>Split.output_1: Close + Split-->>BlockB: Disconnect + Split-->>Split.output_2: Close + Split-->>BlockC: Disconnect diff --git a/lib/protoflow-blocks/src/block_tag.rs b/lib/protoflow-blocks/src/block_tag.rs index 2f9e0bdc..e7015490 100644 --- a/lib/protoflow-blocks/src/block_tag.rs +++ b/lib/protoflow-blocks/src/block_tag.rs @@ -18,6 +18,7 @@ pub enum BlockTag { Drop, Random, // FlowBlocks + Split, // HashBlocks #[cfg(feature = "hash")] Hash, @@ -73,6 +74,7 @@ impl BlockTag { Delay => "Delay", Drop => "Drop", Random => "Random", + Split => "Split", #[cfg(feature = "hash")] Hash => "Hash", Decode => "Decode", @@ -119,6 +121,7 @@ impl FromStr for BlockTag { "Delay" => Delay, "Drop" => Drop, "Random" => Random, + "Split" => Split, #[cfg(feature = "hash")] "Hash" => Hash, "Decode" => Decode, @@ -176,6 +179,7 @@ impl BlockInstantiation for BlockTag { Delay => Box::new(super::Delay::::with_system(system, None)), Drop => Box::new(super::Drop::::with_system(system)), Random => Box::new(super::Random::::with_system(system, None)), + Split => Box::new(super::Split::::with_system(system)), #[cfg(feature = "hash")] Hash => Box::new(super::Hash::with_system(system, None)), Decode => Box::new(super::Decode::::with_system(system, None)), diff --git a/lib/protoflow-blocks/src/blocks/flow.rs b/lib/protoflow-blocks/src/blocks/flow.rs index 21496e25..87d398c4 100644 --- a/lib/protoflow-blocks/src/blocks/flow.rs +++ b/lib/protoflow-blocks/src/blocks/flow.rs @@ -1,30 +1,76 @@ // This is free and unencumbered software released into the public domain. pub mod flow { + use crate::{InputPortName, OutputPortName}; + use super::{ - prelude::{Cow, Named}, - BlockConnections, BlockInstantiation, + prelude::{vec, Box, Cow, Named, Vec}, + BlockConnections, BlockInstantiation, System, }; - pub trait FlowBlocks {} + use protoflow_core::{Block, Message}; + + pub trait FlowBlocks { + fn split + 'static>(&mut self) -> Split; + } #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] #[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] - pub enum FlowBlockTag {} + pub enum FlowBlockTag { + Split, + } #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] #[derive(Clone, Debug)] - pub enum FlowBlockConfig {} + pub enum FlowBlockConfig { + Split { + input: InputPortName, + output_1: OutputPortName, + output_2: OutputPortName, + }, + } impl Named for FlowBlockConfig { fn name(&self) -> Cow { - unreachable!() + use FlowBlockConfig::*; + Cow::Borrowed(match self { + Split { .. } => "Split", + }) } } - impl BlockConnections for FlowBlockConfig {} + impl BlockConnections for FlowBlockConfig { + fn output_connections(&self) -> Vec<(&'static str, Option)> { + use FlowBlockConfig::*; + match self { + Split { + output_1, output_2, .. + } => { + vec![ + ("output_1", Some(output_1.clone())), + ("output_2", Some(output_2.clone())), + ] + } + } + } + } + + impl BlockInstantiation for FlowBlockConfig { + fn instantiate(&self, system: &mut System) -> Box { + use super::SystemBuilding; + use FlowBlockConfig::*; + match self { + Split { .. } => Box::new(super::Split::new( + system.input_any(), + system.output(), + system.output(), + )), + } + } + } - impl BlockInstantiation for FlowBlockConfig {} + mod split; + pub use split::*; } pub use flow::*; diff --git a/lib/protoflow-blocks/src/blocks/flow/split.rs b/lib/protoflow-blocks/src/blocks/flow/split.rs new file mode 100644 index 00000000..f7b7dc91 --- /dev/null +++ b/lib/protoflow-blocks/src/blocks/flow/split.rs @@ -0,0 +1,170 @@ +// This is free and unencumbered software released into the public domain. + +use crate::{FlowBlocks, StdioConfig, StdioError, StdioSystem, SysBlocks, System}; +use protoflow_core::{ + types::Any, Block, BlockResult, BlockRuntime, InputPort, Message, OutputPort, +}; +use protoflow_derive::Block; +use simple_mermaid::mermaid; + +/// Divides a single input message stream into multiple output streams using a round-robin approach. +/// +/// # Block Diagram +#[doc = mermaid!("../../../doc/flow/split.mmd")] +/// +/// # Sequence Diagram +#[doc = mermaid!("../../../doc/flow/split.seq.mmd" framed)] +/// +/// # Examples +/// +/// ## Using the block in a system +/// +/// ```rust +/// # use protoflow_blocks::*; +/// # fn main() { +/// System::build(|s| { +/// let stdin = s.read_stdin(); +/// let split = s.split(); +/// s.connect(&stdin.output, &split.input); +/// let stdout_1 = s.write_stdout(); +/// s.connect(&split.output_1, &stdout_1.input); +/// let stdout_2 = s.write_stdout(); +/// s.connect(&split.output_2, &stdout_2.input); +/// }); +/// # } +/// ``` +/// +/// ## Running the block via the CLI +/// +/// ```console +/// $ protoflow execute Split +/// ``` +/// +#[derive(Block, Clone)] +pub struct Split { + /// The input message stream. + #[input] + pub input: InputPort, + #[output] + pub output_1: OutputPort, + #[output] + pub output_2: OutputPort, + #[state] + pub message_count: u128, +} + +impl Split { + pub fn new(input: InputPort, output_1: OutputPort, output_2: OutputPort) -> Self { + Self { + input, + output_1, + output_2, + message_count: 0, + } + } +} +impl Split { + pub fn with_system(system: &System) -> Self { + use crate::SystemBuilding; + Self::new(system.input(), system.output(), system.output()) + } +} + +impl Block for Split { + fn execute(&mut self, runtime: &dyn BlockRuntime) -> BlockResult { + runtime.wait_for(&self.output_1)?; + runtime.wait_for(&self.output_2)?; + while let Some(message) = self.input.recv()? { + match self.message_count % 2 { + 0 => self.output_1.send(&message)?, + 1 => self.output_2.send(&message)?, + _ => {} + } + self.message_count += 1; + } + Ok(()) + } +} + +#[cfg(feature = "std")] +impl StdioSystem for Split { + fn build_system(config: StdioConfig) -> Result { + use crate::SystemBuilding; + config.reject_any()?; + + Ok(System::build(|s| { + let stdin = s.read_stdin(); + + let split = s.split(); + s.connect(&stdin.output, &split.input); + + let stdout_1 = s.write_stdout(); + s.connect(&split.output_1, &stdout_1.input); + + let stdout_2 = s.write_stdout(); + s.connect(&split.output_2, &stdout_2.input); + })) + } +} + +#[cfg(test)] +pub mod split_tests { + use crate::{CoreBlocks, FlowBlocks, SysBlocks, System}; + use protoflow_core::prelude::String; + use tracing::error; + extern crate std; + + #[test] + fn instantiate_block() { + // Check that the block is constructible: + let _ = System::build(|s| { + let _ = s.split::(); + }); + } + + #[test] + #[ignore = "requires stdin"] + fn run_split_stdout_and_file() { + use super::*; + use protoflow_core::SystemBuilding; + if let Err(e) = System::run(|s| { + let stdin = s.read_stdin(); + let split = s.split(); + s.connect(&stdin.output, &split.input); + + let stdout_1 = s.write_stdout(); + s.connect(&split.output_1, &stdout_1.input); + + let file = s.const_string("text.txt"); + let write_file = s.write_file().with_flags(crate::WriteFlags { + create: true, + append: true, + }); + s.connect(&file.output, &write_file.path); + s.connect(&split.output_2, &write_file.input); + }) { + error!("{}", e) + } + } + + #[test] + #[ignore = "requires stdin"] + fn run_split_to_stdout() { + //use super::*; + use protoflow_core::SystemBuilding; + if let Err(e) = System::run(|s| { + let stdin = s.read_stdin(); + + let split = s.split(); + s.connect(&stdin.output, &split.input); + + let stdout_1 = s.write_stdout(); + s.connect(&split.output_1, &stdout_1.input); + + let stdout_2 = s.write_stdout(); + s.connect(&split.output_2, &stdout_2.input); + }) { + error!("{}", e) + } + } +} diff --git a/lib/protoflow-blocks/src/lib.rs b/lib/protoflow-blocks/src/lib.rs index 52481486..57ade69c 100644 --- a/lib/protoflow-blocks/src/lib.rs +++ b/lib/protoflow-blocks/src/lib.rs @@ -60,6 +60,7 @@ pub fn build_stdio_system( "Drop" => Drop::::build_system(config)?, "Random" => Random::::build_system(config)?, // FlowBlocks + "Split" => Split::::build_system(config)?, // HashBlocks #[cfg(feature = "hash")] "Hash" => Hash::build_system(config)?, diff --git a/lib/protoflow-blocks/src/system.rs b/lib/protoflow-blocks/src/system.rs index 8bc58232..9ed3c5f2 100644 --- a/lib/protoflow-blocks/src/system.rs +++ b/lib/protoflow-blocks/src/system.rs @@ -7,8 +7,8 @@ use crate::{ types::{DelayType, Encoding}, AllBlocks, Buffer, ConcatStrings, Const, CoreBlocks, Count, Decode, DecodeCsv, DecodeHex, DecodeJson, Delay, Drop, Encode, EncodeCsv, EncodeHex, EncodeJson, FlowBlocks, HashBlocks, - IoBlocks, MathBlocks, Random, ReadDir, ReadEnv, ReadFile, ReadSocket, ReadStdin, SplitString, - SysBlocks, TextBlocks, WriteFile, WriteSocket, WriteStderr, WriteStdout, + IoBlocks, MathBlocks, Random, ReadDir, ReadEnv, ReadFile, ReadSocket, ReadStdin, Split, + SplitString, SysBlocks, TextBlocks, WriteFile, WriteSocket, WriteStderr, WriteStdout, }; use protoflow_core::{ Block, BlockID, BlockResult, BoxedBlockType, InputPort, Message, OutputPort, PortID, @@ -158,7 +158,11 @@ impl CoreBlocks for System { } } -impl FlowBlocks for System {} +impl FlowBlocks for System { + fn split + 'static>(&mut self) -> Split { + self.0.block(Split::::with_system(self)) + } +} #[cfg(not(feature = "hash"))] impl HashBlocks for System {} From a6b2ca1e54ae488d5602206265ffb1115aca0a2f Mon Sep 17 00:00:00 2001 From: evren Date: Sat, 23 Nov 2024 11:40:00 +0300 Subject: [PATCH 02/24] wip --- README.md | 6 + lib/protoflow-blocks/src/block_config.rs | 12 +- lib/protoflow-blocks/src/block_tag.rs | 21 +- lib/protoflow-blocks/src/blocks/flow.rs | 62 ++++++ .../src/blocks/flow/concat.rs | 192 ++++++++++++++++++ .../src/blocks/flow/replicate.rs | 166 +++++++++++++++ lib/protoflow-blocks/src/blocks/flow/sort.rs | 152 ++++++++++++++ lib/protoflow-blocks/src/lib.rs | 3 + lib/protoflow-blocks/src/system.rs | 21 +- 9 files changed, 619 insertions(+), 16 deletions(-) create mode 100644 lib/protoflow-blocks/src/blocks/flow/concat.rs create mode 100644 lib/protoflow-blocks/src/blocks/flow/replicate.rs create mode 100644 lib/protoflow-blocks/src/blocks/flow/sort.rs diff --git a/README.md b/README.md index 7e71d511..5e317314 100644 --- a/README.md +++ b/README.md @@ -114,6 +114,7 @@ The built-in blocks provided by Protoflow are listed below: |:------------------|:-------------------------------------------------------------------------------------------------------------------------------| | [`Buffer`] | Stores all messages it receives. | | [`ConcatStrings`] | Concatenates the received string messages, with an optional delimiter string inserted between each message. | +| [`Concat`] | Merges multiple input message streams into a single output stream. | | [`Const`] | Sends a constant value. | | [`Count`] | Counts the number of messages it receives, while optionally passing them through. | | [`Decode`] | Decodes messages from a byte stream. | @@ -133,6 +134,8 @@ The built-in blocks provided by Protoflow are listed below: | [`ReadFile`] | Reads bytes from the contents of a file. | | [`ReadSocket`] | Reads bytes from a TCP socket. | | [`ReadStdin`] | Reads bytes from standard input (aka stdin). | +| [`Replicate`] | Duplicates a single input message stream into multiple identical output streams. | +| [`Sort`] | Sorts a single input message stream in ascending order. | | [`Split`] | Divides a single input message stream into multiple output streams using a round-robin approach. | | [`SplitString`] | Splits the received input message, with an optional delimiter string parameter. | | [`WriteFile`] | Writes or appends bytes to the contents of a file. | @@ -822,6 +825,7 @@ To add a new block type implementation, make sure to examine and amend: [`examples`]: lib/protoflow/examples [`Buffer`]: https://docs.rs/protoflow-blocks/latest/protoflow_blocks/struct.Buffer.html +[`Concat`]: https://docs.rs/protoflow-blocks/latest/protoflow_blocks/struct.Concat.html [`ConcatStrings`]: https://docs.rs/protoflow-blocks/latest/protoflow_blocks/struct.ConcatStrings.html [`Const`]: https://docs.rs/protoflow-blocks/latest/protoflow_blocks/struct.Const.html [`Count`]: https://docs.rs/protoflow-blocks/latest/protoflow_blocks/struct.Count.html @@ -842,6 +846,8 @@ To add a new block type implementation, make sure to examine and amend: [`ReadFile`]: https://docs.rs/protoflow-blocks/latest/protoflow_blocks/struct.ReadFile.html [`ReadSocket`]: https://docs.rs/protoflow-blocks/latest/protoflow_blocks/struct.ReadSocket.html [`ReadStdin`]: https://docs.rs/protoflow-blocks/latest/protoflow_blocks/struct.ReadStdin.html +[`Replicate`]: https://docs.rs/protoflow-blocks/latest/protoflow_blocks/struct.Replicate.html +[`Sort`]: https://docs.rs/protoflow-blocks/latest/protoflow_blocks/struct.Sort.html [`Split`]: https://docs.rs/protoflow-blocks/latest/protoflow_blocks/struct.Split.html [`SplitString`]: https://docs.rs/protoflow-blocks/latest/protoflow_blocks/struct.SplitString.html [`WriteFile`]: https://docs.rs/protoflow-blocks/latest/protoflow_blocks/struct.WriteFile.html diff --git a/lib/protoflow-blocks/src/block_config.rs b/lib/protoflow-blocks/src/block_config.rs index 8e32c0f0..fbe3ace3 100644 --- a/lib/protoflow-blocks/src/block_config.rs +++ b/lib/protoflow-blocks/src/block_config.rs @@ -30,12 +30,6 @@ pub enum BlockConfig { feature = "hash-sha1", feature = "hash-sha2" ))] - #[cfg(any( - feature = "hash-blake3", - feature = "hash-md5", - feature = "hash-sha1", - feature = "hash-sha2" - ))] Hash(HashBlockConfig), Io(IoBlockConfig), Math(MathBlockConfig), @@ -63,6 +57,12 @@ impl<'de> serde::Deserialize<'de> for BlockConfig { .unwrap() } + "Concat" | "Replicate" | "Split" | "Sort" => { + FlowBlockConfig::deserialize(value.clone()) + .map(BlockConfig::Flow) + .unwrap() + } + #[cfg(any( feature = "hash-blake3", feature = "hash-md5", diff --git a/lib/protoflow-blocks/src/block_tag.rs b/lib/protoflow-blocks/src/block_tag.rs index 0f0a0588..9337983d 100644 --- a/lib/protoflow-blocks/src/block_tag.rs +++ b/lib/protoflow-blocks/src/block_tag.rs @@ -18,6 +18,9 @@ pub enum BlockTag { Drop, Random, // FlowBlocks + Concat, + Replicate, + Sort, Split, // HashBlocks #[cfg(any( @@ -74,13 +77,12 @@ impl BlockTag { use BlockTag::*; match self { Buffer => "Buffer", + Concat => "Concat", Const => "Const", Count => "Count", Delay => "Delay", Drop => "Drop", Random => "Random", - Split => "Split", - #[cfg(feature = "hash")] #[cfg(any( feature = "hash-blake3", feature = "hash-md5", @@ -104,6 +106,9 @@ impl BlockTag { ReadSocket => "ReadSocket", #[cfg(feature = "std")] ReadStdin => "ReadStdin", + Replicate => "Replicate", + Sort => "Sort", + Split => "Split", #[cfg(feature = "std")] WriteFile => "WriteFile", #[cfg(feature = "std")] @@ -127,13 +132,12 @@ impl FromStr for BlockTag { use BlockTag::*; Ok(match input { "Buffer" => Buffer, + "Concat" => Concat, "Const" => Const, "Count" => Count, "Delay" => Delay, "Drop" => Drop, "Random" => Random, - "Split" => Split, - #[cfg(feature = "hash")] #[cfg(any( feature = "hash-blake3", feature = "hash-md5", @@ -157,6 +161,9 @@ impl FromStr for BlockTag { "ReadSocket" => ReadSocket, #[cfg(feature = "std")] "ReadStdin" => ReadStdin, + "Replicate" => Replicate, + "Sort" => Sort, + "Split" => Split, #[cfg(feature = "std")] "WriteFile" => WriteFile, #[cfg(feature = "std")] @@ -191,13 +198,12 @@ impl BlockInstantiation for BlockTag { use BlockTag::*; match self { Buffer => Box::new(super::Buffer::::with_system(system)), + Concat => Box::new(super::Concat::::with_system(system)), Const => Box::new(super::Const::::with_system(system, String::new())), Count => Box::new(super::Count::::with_system(system)), Delay => Box::new(super::Delay::::with_system(system, None)), Drop => Box::new(super::Drop::::with_system(system)), Random => Box::new(super::Random::::with_system(system, None)), - Split => Box::new(super::Split::::with_system(system)), - #[cfg(feature = "hash")] #[cfg(any( feature = "hash-blake3", feature = "hash-md5", @@ -221,6 +227,9 @@ impl BlockInstantiation for BlockTag { ReadSocket => Box::new(super::ReadSocket::with_system(system, None)), #[cfg(feature = "std")] ReadStdin => Box::new(super::ReadStdin::with_system(system, None)), + Replicate => Box::new(super::Replicate::::with_system(system)), + Sort => Box::new(super::Sort::::with_system(system)), + Split => Box::new(super::Split::::with_system(system)), #[cfg(feature = "std")] WriteFile => Box::new(super::WriteFile::with_system(system, None)), #[cfg(feature = "std")] diff --git a/lib/protoflow-blocks/src/blocks/flow.rs b/lib/protoflow-blocks/src/blocks/flow.rs index 87d398c4..c0ffea96 100644 --- a/lib/protoflow-blocks/src/blocks/flow.rs +++ b/lib/protoflow-blocks/src/blocks/flow.rs @@ -11,18 +11,39 @@ pub mod flow { use protoflow_core::{Block, Message}; pub trait FlowBlocks { + fn concat + 'static>(&mut self) -> Concat; + fn replicate + 'static>(&mut self) -> Replicate; + fn sort + PartialOrd + 'static>(&mut self) -> Sort; fn split + 'static>(&mut self) -> Split; } #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] #[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] pub enum FlowBlockTag { + Concat, + Replicate, + Sort, Split, } #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] #[derive(Clone, Debug)] pub enum FlowBlockConfig { + Concat { + input_1: InputPortName, + input_2: InputPortName, + output: OutputPortName, + }, + Replicate { + input: InputPortName, + output_1: OutputPortName, + output_2: OutputPortName, + }, + Sort { + input: InputPortName, + stop: InputPortName, + output: OutputPortName, + }, Split { input: InputPortName, output_1: OutputPortName, @@ -34,6 +55,9 @@ pub mod flow { fn name(&self) -> Cow { use FlowBlockConfig::*; Cow::Borrowed(match self { + Concat { .. } => "Concat", + Replicate { .. } => "Replicate", + Sort { .. } => "Sort", Split { .. } => "Split", }) } @@ -43,6 +67,20 @@ pub mod flow { fn output_connections(&self) -> Vec<(&'static str, Option)> { use FlowBlockConfig::*; match self { + Concat { output, .. } => { + vec![("output", Some(output.clone()))] + } + Replicate { + output_1, output_2, .. + } => { + vec![ + ("output_1", Some(output_1.clone())), + ("output_2", Some(output_2.clone())), + ] + } + Sort { output, .. } => { + vec![("output", Some(output.clone()))] + } Split { output_1, output_2, .. } => { @@ -60,6 +98,21 @@ pub mod flow { use super::SystemBuilding; use FlowBlockConfig::*; match self { + Concat { .. } => Box::new(super::Concat::new( + system.input_any(), + system.input_any(), + system.output(), + )), + Replicate { .. } => Box::new(super::Replicate::new( + system.input_any(), + system.output(), + system.output(), + )), + Sort { .. } => Box::new(super::Sort::new( + system.input_any(), + system.input(), + system.output(), + )), Split { .. } => Box::new(super::Split::new( system.input_any(), system.output(), @@ -69,6 +122,15 @@ pub mod flow { } } + mod concat; + pub use concat::*; + + mod replicate; + pub use replicate::*; + + mod sort; + pub use sort::*; + mod split; pub use split::*; } diff --git a/lib/protoflow-blocks/src/blocks/flow/concat.rs b/lib/protoflow-blocks/src/blocks/flow/concat.rs new file mode 100644 index 00000000..2c6e38ee --- /dev/null +++ b/lib/protoflow-blocks/src/blocks/flow/concat.rs @@ -0,0 +1,192 @@ +// This is free and unencumbered software released into the public domain. +extern crate std; + +use crate::{FlowBlocks, StdioConfig, StdioError, StdioSystem, SysBlocks, System}; +use protoflow_core::{ + types::Any, Block, BlockResult, BlockRuntime, InputPort, Message, OutputPort, +}; +use protoflow_derive::Block; +use simple_mermaid::mermaid; +use std::sync::Arc; + +/// Divides a single input message stream into multiple output streams using a round-robin approach. +/// +/// # Block Diagram +#[doc = mermaid!("../../../doc/flow/split.mmd")] +/// +/// # Sequence Diagram +#[doc = mermaid!("../../../doc/flow/split.seq.mmd" framed)] +/// +/// # Examples +/// +/// ## Using the block in a system +/// +/// ```rust +/// # use protoflow_blocks::*; +/// # fn main() { +/// System::build(|s| { +/// let stdin = s.read_stdin(); +/// let split = s.split(); +/// s.connect(&stdin.output, &split.input); +/// let stdout_1 = s.write_stdout(); +/// s.connect(&split.output_1, &stdout_1.input); +/// let stdout_2 = s.write_stdout(); +/// s.connect(&split.output_2, &stdout_2.input); +/// }); +/// # } +/// ``` +/// +/// ## Running the block via the CLI +/// +/// ```console +/// $ protoflow execute Concat +/// ``` +/// +#[derive(Block, Clone)] +pub struct Concat { + /// The input message stream. + #[input] + pub input_1: InputPort, + #[output] + pub input_2: InputPort, + #[output] + pub output: OutputPort, +} + +impl Concat { + pub fn new(input_1: InputPort, input_2: InputPort, output: OutputPort) -> Self { + Self { + input_1, + input_2, + output, + } + } +} +impl Concat { + pub fn with_system(system: &System) -> Self { + use crate::SystemBuilding; + Self::new(system.input(), system.input(), system.output()) + } +} + +impl Block for Concat { + fn execute(&mut self, runtime: &dyn BlockRuntime) -> BlockResult { + runtime.wait_for(&self.output)?; + + // Wrap inputs and outputs in Arc for shared ownership + let input1 = Arc::new(self.input_1.clone()); + let input2 = Arc::new(self.input_2.clone()); + let output = Arc::new(self.output.clone()); + + // Spawn thread for input1 + let input1_clone = Arc::clone(&input1); + let output_clone = Arc::clone(&output); + let handle1 = std::thread::spawn(move || { + while let Ok(Some(message)) = input1_clone.recv() { + output_clone.send(&message).unwrap(); + } + }); + + // Spawn thread for input2 + let input2_clone = Arc::clone(&input2); + let output_clone2 = Arc::clone(&output); + let handle2 = std::thread::spawn(move || { + while let Ok(Some(message)) = input2_clone.recv() { + output_clone2.send(&message).unwrap(); + } + }); + + // Wait for both threads to finish + handle1.join().unwrap(); + handle2.join().unwrap(); + + Ok(()) + } +} + +#[cfg(feature = "std")] +impl StdioSystem for Concat { + fn build_system(config: StdioConfig) -> Result { + use crate::SystemBuilding; + config.reject_any()?; + + Ok(System::build(|s| { + let stdin = s.read_stdin(); + + let split = s.split(); + s.connect(&stdin.output, &split.input); + + let concat = s.block(Concat::new(s.input(), s.input(), s.output())); + + s.connect(&split.output_1, &concat.input_1); + s.connect(&split.output_2, &concat.input_2); + + let stdout_1 = s.write_stdout(); + s.connect(&concat.output, &stdout_1.input); + })) + } +} + +#[cfg(test)] +pub mod split_tests { + use crate::{Concat, CoreBlocks, FlowBlocks, StdioSystem, SysBlocks, System}; + use protoflow_core::prelude::String; + use tracing::error; + extern crate std; + + #[test] + fn instantiate_block() { + // Check that the block is constructible: + let _ = System::build(|s| { + let _ = s.split::(); + }); + } + + #[test] + #[ignore = "requires stdin"] + fn run_split_stdout_and_file() { + use super::*; + use protoflow_core::SystemBuilding; + if let Err(e) = System::run(|s| { + let stdin = s.read_stdin(); + let split = s.split(); + s.connect(&stdin.output, &split.input); + + let stdout_1 = s.write_stdout(); + s.connect(&split.output_1, &stdout_1.input); + + let file = s.const_string("text.txt"); + let write_file = s.write_file().with_flags(crate::WriteFlags { + create: true, + append: true, + }); + s.connect(&file.output, &write_file.path); + s.connect(&split.output_2, &write_file.input); + }) { + error!("{}", e) + } + } + + #[test] + #[ignore = "requires stdin"] + fn run_split_to_stdout() { + //use super::*; + use protoflow_core::SystemBuilding; + if let Err(e) = System::run(|s| { + let stdin = s.read_stdin(); + + let split = s.split(); + s.connect(&stdin.output, &split.input); + + let concat = s.block(Concat::new(s.input(), s.input(), s.output())); + + s.connect(&split.output_1, &concat.input_1); + s.connect(&split.output_2, &concat.input_2); + + let stdout_1 = s.write_stdout(); + s.connect(&concat.output, &stdout_1.input); + }) { + error!("{}", e) + } + } +} diff --git a/lib/protoflow-blocks/src/blocks/flow/replicate.rs b/lib/protoflow-blocks/src/blocks/flow/replicate.rs new file mode 100644 index 00000000..a1a38627 --- /dev/null +++ b/lib/protoflow-blocks/src/blocks/flow/replicate.rs @@ -0,0 +1,166 @@ +// This is free and unencumbered software released into the public domain. + +use crate::{StdioConfig, StdioError, StdioSystem, SysBlocks, System}; +use protoflow_core::{ + types::Any, Block, BlockResult, BlockRuntime, InputPort, Message, OutputPort, +}; +use protoflow_derive::Block; + +/// Divides a single input message stream into multiple output streams using a round-robin approach. +/// +/// # Block Diagram + +/// +/// # Sequence Diagram + +/// +/// # Examples +/// +/// ## Using the block in a system +/// +/// ```rust +/// # use protoflow_blocks::*; +/// # fn main() { +/// System::build(|s| { +/// let stdin = s.read_stdin(); +/// let replicate = s.replicate(); +/// s.connect(&stdin.output, &replicate.input); +/// let stdout_1 = s.write_stdout(); +/// s.connect(&replicate.output_1, &stdout_1.input); +/// let stdout_2 = s.write_stdout(); +/// s.connect(&replicate.output_2, &stdout_2.input); +/// }); +/// # } +/// ``` +/// +/// ## Running the block via the CLI +/// +/// ```console +/// $ protoflow execute Replicate +/// ``` +/// +#[derive(Block, Clone)] +pub struct Replicate { + /// The input message stream. + #[input] + pub input: InputPort, + #[output] + pub output_1: OutputPort, + #[output] + pub output_2: OutputPort, +} + +impl Replicate { + pub fn new(input: InputPort, output_1: OutputPort, output_2: OutputPort) -> Self { + Self { + input, + output_1, + output_2, + } + } +} +impl Replicate { + pub fn with_system(system: &System) -> Self { + use crate::SystemBuilding; + Self::new(system.input(), system.output(), system.output()) + } +} + +impl Block for Replicate { + fn execute(&mut self, runtime: &dyn BlockRuntime) -> BlockResult { + runtime.wait_for(&self.output_1)?; + runtime.wait_for(&self.output_2)?; + + while let Some(message) = self.input.recv()? { + self.output_1.send(&message)?; + self.output_2.send(&message)?; + } + + Ok(()) + } +} + +#[cfg(feature = "std")] +impl StdioSystem for Replicate { + fn build_system(config: StdioConfig) -> Result { + use crate::SystemBuilding; + config.reject_any()?; + + Ok(System::build(|s| { + let stdin = s.read_stdin(); + + let replicate = s.block(Replicate::new(s.input(), s.output(), s.output())); + s.connect(&stdin.output, &replicate.input); + + let stdout_1 = s.write_stdout(); + s.connect(&replicate.output_1, &stdout_1.input); + + let stdout_2 = s.write_stdout(); + s.connect(&replicate.output_2, &stdout_2.input); + })) + } +} + +#[cfg(test)] +pub mod replicate_tests { + use crate::{CoreBlocks, FlowBlocks, SysBlocks, System}; + use protoflow_core::{prelude::String, SystemBuilding}; + use tracing::error; + + use super::Replicate; + extern crate std; + + #[test] + fn instantiate_block() { + // Check that the block is constructible: + let _ = System::build(|s| { + let _ = s.block(Replicate::new(s.input_any(), s.output(), s.output())); + }); + } + + #[test] + #[ignore = "requires stdin"] + fn run_replicate_stdout_and_file() { + use super::*; + use protoflow_core::SystemBuilding; + if let Err(e) = System::run(|s| { + let stdin = s.read_stdin(); + let replicate = s.block(Replicate::new(s.input(), s.output_bytes(), s.output())); + s.connect(&stdin.output, &replicate.input); + + let stdout_1 = s.write_stdout(); + s.connect(&replicate.output_1, &stdout_1.input); + + let file = s.const_string("text.txt"); + let write_file = s.write_file().with_flags(crate::WriteFlags { + create: true, + append: true, + }); + s.connect(&file.output, &write_file.path); + s.connect(&replicate.output_2, &write_file.input); + }) { + error!("{}", e) + } + } + + #[test] + #[ignore = "requires stdin"] + fn run_replicate_to_stdout() { + //use super::*; + use protoflow_core::SystemBuilding; + if let Err(e) = System::run(|s| { + let stdin = s.read_stdin(); + + let replicate = s.block(Replicate::new(s.input(), s.output_bytes(), s.output())); + s.connect(&stdin.output, &replicate.input); + + let stdout_1 = s.write_stdout(); + s.connect(&replicate.output_1, &stdout_1.input); + + let stdout_2 = s.write_stdout(); + s.connect(&replicate.output_2, &stdout_2.input); + }) { + error!("{}", e) + } + } +} diff --git a/lib/protoflow-blocks/src/blocks/flow/sort.rs b/lib/protoflow-blocks/src/blocks/flow/sort.rs new file mode 100644 index 00000000..f0b0acd8 --- /dev/null +++ b/lib/protoflow-blocks/src/blocks/flow/sort.rs @@ -0,0 +1,152 @@ +// This is free and unencumbered software released into the public domain. + +use crate::{StdioConfig, StdioError, StdioSystem, System}; +use protoflow_core::{ + prelude::Vec, types::Any, Block, BlockResult, BlockRuntime, InputPort, Message, OutputPort, +}; +use protoflow_derive::Block; + +/// A block that simply stores all messages it receives. +/// +/// # Block Diagram + +/// +/// # Sequence Diagram + +/// +/// # Examples +/// +/// ## Using the block in a system +/// +/// ```rust +/// # use protoflow_blocks::*; +/// # fn main() { +/// System::build(|s| { +/// let stdin = s.read_stdin(); +/// let sort = s.sort(); +/// s.connect(&stdin.output, &sort.input); +/// }); +/// # } +/// ``` +/// +/// ## Running the block via the CLI +/// +/// ```console +/// $ protoflow execute Sort +/// ``` +/// +#[derive(Block, Clone)] +pub struct Sort { + /// The input message stream. + #[input] + pub input: InputPort, + + #[input] + pub stop: InputPort, + + /// The output message stream. + #[output] + pub output: OutputPort, + + /// The internal state storing the messages received. + #[state] + messages: Vec, +} + +impl Sort { + pub fn new(input: InputPort, stop: InputPort, output: OutputPort) -> Self { + Self { + input, + stop, + output, + messages: Vec::new(), + } + } + + pub fn messages(&self) -> &Vec { + &self.messages + } +} + +impl Sort { + pub fn with_system(system: &System) -> Self { + use crate::SystemBuilding; + Self::new(system.input(), system.input(), system.output()) + } +} + +impl Block for Sort { + fn execute(&mut self, _runtime: &dyn BlockRuntime) -> BlockResult { + // if let Some(x) = self.stop.recv()? { + // while let Some(message) = self.input.recv()? { + // if message == x { + // self.messages.sort_by(|x, y| x.partial_cmp(y).unwrap()); + // for x in self.messages.iter() { + // self.output.send(x)?; + // } + // self.messages.clear(); + // } else { + // self.messages.push(message); + // } + // } + // } + Ok(()) + } +} + +#[cfg(feature = "std")] +impl StdioSystem for Sort { + fn build_system(config: StdioConfig) -> Result { + use crate::SystemBuilding; + + config.reject_any()?; + + Ok(System::build(|s| { + let stdin = config.read_stdin(s); + let sort = s.block(Sort::new(s.input(), s.input(), s.output())); + s.connect(&stdin.output, &sort.input); + })) + } +} + +#[cfg(test)] +mod tests { + use super::Sort; + use crate::{System, SystemBuilding}; + + #[test] + fn instantiate_block() { + // Check that the block is constructible: + let _ = System::build(|s| { + let _ = s.block(Sort::::new(s.input(), s.input(), s.output())); + }); + } +} + +#[cfg(test)] +pub mod split_tests { + use bytes::Bytes; + use tracing::error; + + use crate::{Const, SysBlocks}; + + #[test] + #[ignore = "requires stdin"] + fn run_split_stdout_and_file() { + use super::*; + use protoflow_core::SystemBuilding; + if let Err(e) = System::run(|s| { + let stdin = s.read_stdin(); + let split = s.block(Sort::new(s.input(), s.input(), s.output())); + s.connect(&stdin.output, &split.input); + + let constant = s.block(Const::::with_params(s.output(), Bytes::from("\n"))); + s.connect(&constant.output, &split.stop); + + let stdout_1 = s.write_stdout(); + s.connect(&split.output, &stdout_1.input); + }) { + error!("{}", e) + } + } +} diff --git a/lib/protoflow-blocks/src/lib.rs b/lib/protoflow-blocks/src/lib.rs index 68506d11..0356b5fa 100644 --- a/lib/protoflow-blocks/src/lib.rs +++ b/lib/protoflow-blocks/src/lib.rs @@ -60,6 +60,9 @@ pub fn build_stdio_system( "Drop" => Drop::::build_system(config)?, "Random" => Random::::build_system(config)?, // FlowBlocks + "Concat" => Concat::::build_system(config)?, + "Replicate" => Replicate::::build_system(config)?, + "Sort" => Sort::::build_system(config)?, "Split" => Split::::build_system(config)?, // HashBlocks #[cfg(any( diff --git a/lib/protoflow-blocks/src/system.rs b/lib/protoflow-blocks/src/system.rs index 514b556c..bba614ba 100644 --- a/lib/protoflow-blocks/src/system.rs +++ b/lib/protoflow-blocks/src/system.rs @@ -5,10 +5,11 @@ use crate::{ prelude::{fmt, Arc, Box, FromStr, Rc, String, ToString}, types::{DelayType, Encoding}, - AllBlocks, Buffer, ConcatStrings, Const, CoreBlocks, Count, Decode, DecodeCsv, DecodeHex, - DecodeJson, Delay, Drop, Encode, EncodeCsv, EncodeHex, EncodeJson, FlowBlocks, HashBlocks, - IoBlocks, MathBlocks, Random, ReadDir, ReadEnv, ReadFile, ReadSocket, ReadStdin, Split, - SplitString, SysBlocks, TextBlocks, WriteFile, WriteSocket, WriteStderr, WriteStdout, + AllBlocks, Buffer, Concat, ConcatStrings, Const, CoreBlocks, Count, Decode, DecodeCsv, + DecodeHex, DecodeJson, Delay, Drop, Encode, EncodeCsv, EncodeHex, EncodeJson, FlowBlocks, + HashBlocks, IoBlocks, MathBlocks, Random, ReadDir, ReadEnv, ReadFile, ReadSocket, ReadStdin, + Replicate, Sort, Split, SplitString, SysBlocks, TextBlocks, WriteFile, WriteSocket, + WriteStderr, WriteStdout, }; use protoflow_core::{ Block, BlockID, BlockResult, BoxedBlockType, InputPort, Message, OutputPort, PortID, @@ -164,6 +165,18 @@ impl CoreBlocks for System { } impl FlowBlocks for System { + fn concat + 'static>(&mut self) -> Concat { + self.0.block(Concat::::with_system(self)) + } + + fn replicate + 'static>(&mut self) -> Replicate { + self.0.block(Replicate::::with_system(self)) + } + + fn sort + PartialOrd + 'static>(&mut self) -> Sort { + self.0.block(Sort::::with_system(self)) + } + fn split + 'static>(&mut self) -> Split { self.0.block(Split::::with_system(self)) } From fcbe544147390a0409d78c30b0ab8bbec0666dc6 Mon Sep 17 00:00:00 2001 From: evren Date: Sat, 23 Nov 2024 16:35:25 +0300 Subject: [PATCH 03/24] remove public test modules --- lib/protoflow-blocks/src/blocks/flow/concat.rs | 2 +- lib/protoflow-blocks/src/blocks/flow/replicate.rs | 2 +- lib/protoflow-blocks/src/blocks/flow/sort.rs | 2 +- lib/protoflow-blocks/src/blocks/flow/split.rs | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/protoflow-blocks/src/blocks/flow/concat.rs b/lib/protoflow-blocks/src/blocks/flow/concat.rs index 2c6e38ee..e440fd88 100644 --- a/lib/protoflow-blocks/src/blocks/flow/concat.rs +++ b/lib/protoflow-blocks/src/blocks/flow/concat.rs @@ -128,7 +128,7 @@ impl StdioSystem for Concat { } #[cfg(test)] -pub mod split_tests { +mod split_tests { use crate::{Concat, CoreBlocks, FlowBlocks, StdioSystem, SysBlocks, System}; use protoflow_core::prelude::String; use tracing::error; diff --git a/lib/protoflow-blocks/src/blocks/flow/replicate.rs b/lib/protoflow-blocks/src/blocks/flow/replicate.rs index a1a38627..870a15e1 100644 --- a/lib/protoflow-blocks/src/blocks/flow/replicate.rs +++ b/lib/protoflow-blocks/src/blocks/flow/replicate.rs @@ -102,7 +102,7 @@ impl StdioSystem for Replicate { } #[cfg(test)] -pub mod replicate_tests { +mod replicate_tests { use crate::{CoreBlocks, FlowBlocks, SysBlocks, System}; use protoflow_core::{prelude::String, SystemBuilding}; use tracing::error; diff --git a/lib/protoflow-blocks/src/blocks/flow/sort.rs b/lib/protoflow-blocks/src/blocks/flow/sort.rs index f0b0acd8..51e859ec 100644 --- a/lib/protoflow-blocks/src/blocks/flow/sort.rs +++ b/lib/protoflow-blocks/src/blocks/flow/sort.rs @@ -124,7 +124,7 @@ mod tests { } #[cfg(test)] -pub mod split_tests { +mod split_tests { use bytes::Bytes; use tracing::error; diff --git a/lib/protoflow-blocks/src/blocks/flow/split.rs b/lib/protoflow-blocks/src/blocks/flow/split.rs index f7b7dc91..66707036 100644 --- a/lib/protoflow-blocks/src/blocks/flow/split.rs +++ b/lib/protoflow-blocks/src/blocks/flow/split.rs @@ -108,7 +108,7 @@ impl StdioSystem for Split { } #[cfg(test)] -pub mod split_tests { +mod split_tests { use crate::{CoreBlocks, FlowBlocks, SysBlocks, System}; use protoflow_core::prelude::String; use tracing::error; From 0684169f0e9a506f14475ea751958963f4fac2f0 Mon Sep 17 00:00:00 2001 From: evren Date: Mon, 25 Nov 2024 09:10:16 +0300 Subject: [PATCH 04/24] concat fix --- .../src/blocks/flow/concat.rs | 30 +++++++++++-------- 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/lib/protoflow-blocks/src/blocks/flow/concat.rs b/lib/protoflow-blocks/src/blocks/flow/concat.rs index e440fd88..8a119e2e 100644 --- a/lib/protoflow-blocks/src/blocks/flow/concat.rs +++ b/lib/protoflow-blocks/src/blocks/flow/concat.rs @@ -1,13 +1,15 @@ // This is free and unencumbered software released into the public domain. extern crate std; +use core::ops::DerefMut; + +use crate::prelude::{Arc, Vec}; use crate::{FlowBlocks, StdioConfig, StdioError, StdioSystem, SysBlocks, System}; use protoflow_core::{ types::Any, Block, BlockResult, BlockRuntime, InputPort, Message, OutputPort, }; use protoflow_derive::Block; use simple_mermaid::mermaid; -use std::sync::Arc; /// Divides a single input message stream into multiple output streams using a round-robin approach. /// @@ -73,33 +75,37 @@ impl Block for Concat { fn execute(&mut self, runtime: &dyn BlockRuntime) -> BlockResult { runtime.wait_for(&self.output)?; - // Wrap inputs and outputs in Arc for shared ownership + let mut buffer1 = Vec::new(); + let mut buffer2 = Vec::new(); + let input1 = Arc::new(self.input_1.clone()); let input2 = Arc::new(self.input_2.clone()); - let output = Arc::new(self.output.clone()); - // Spawn thread for input1 let input1_clone = Arc::clone(&input1); - let output_clone = Arc::clone(&output); let handle1 = std::thread::spawn(move || { while let Ok(Some(message)) = input1_clone.recv() { - output_clone.send(&message).unwrap(); + buffer1.push(message); } + buffer1 }); - // Spawn thread for input2 let input2_clone = Arc::clone(&input2); - let output_clone2 = Arc::clone(&output); let handle2 = std::thread::spawn(move || { while let Ok(Some(message)) = input2_clone.recv() { - output_clone2.send(&message).unwrap(); + buffer2.push(message); } + buffer2 }); - // Wait for both threads to finish - handle1.join().unwrap(); - handle2.join().unwrap(); + let buffer1 = handle1.join().unwrap(); + let buffer2 = handle2.join().unwrap(); + let mut combined = buffer1; + combined.extend(buffer2); + + for message in combined { + self.output.send(&message)?; + } Ok(()) } } From 3f9d2cb182afdeee7c09446ab825e46d24ca6efd Mon Sep 17 00:00:00 2001 From: evren Date: Tue, 26 Nov 2024 13:15:10 +0300 Subject: [PATCH 05/24] wip --- lib/protoflow-blocks/doc/flow/concat.mmd | 15 ++ lib/protoflow-blocks/doc/flow/concat.seq.mmd | 28 +++ lib/protoflow-blocks/doc/flow/replicate.mmd | 17 ++ .../doc/flow/replicate.seq.mmd | 27 +++ lib/protoflow-blocks/doc/flow/sort.mmd | 11 ++ lib/protoflow-blocks/doc/flow/sort.seq.mmd | 22 +++ .../src/blocks/flow/concat.rs | 166 ++++++++---------- .../src/blocks/flow/replicate.rs | 68 ++----- lib/protoflow-blocks/src/blocks/flow/sort.rs | 7 +- lib/protoflow-blocks/src/blocks/flow/split.rs | 4 +- 10 files changed, 217 insertions(+), 148 deletions(-) create mode 100644 lib/protoflow-blocks/doc/flow/concat.mmd create mode 100644 lib/protoflow-blocks/doc/flow/concat.seq.mmd create mode 100644 lib/protoflow-blocks/doc/flow/replicate.mmd create mode 100644 lib/protoflow-blocks/doc/flow/replicate.seq.mmd create mode 100644 lib/protoflow-blocks/doc/flow/sort.mmd create mode 100644 lib/protoflow-blocks/doc/flow/sort.seq.mmd diff --git a/lib/protoflow-blocks/doc/flow/concat.mmd b/lib/protoflow-blocks/doc/flow/concat.mmd new file mode 100644 index 00000000..a5108805 --- /dev/null +++ b/lib/protoflow-blocks/doc/flow/concat.mmd @@ -0,0 +1,15 @@ +block-beta + columns 7 + space:1 Source1 space:5 + space:3 Concat space:1 Sink space:1 + space:1 Source2 space:5 + Source1-- "input" -->Concat + Source2-- "input" -->Concat + Concat-- "output" -->Sink + + classDef block height:48px,padding:8px; + classDef hidden visibility:none; + class Concat block + class Source1 hidden + class Source2 hidden + class Sink hidden diff --git a/lib/protoflow-blocks/doc/flow/concat.seq.mmd b/lib/protoflow-blocks/doc/flow/concat.seq.mmd new file mode 100644 index 00000000..bde6827e --- /dev/null +++ b/lib/protoflow-blocks/doc/flow/concat.seq.mmd @@ -0,0 +1,28 @@ +sequenceDiagram + autonumber + participant BlockA as Another block + participant BlockA2 as Another block + participant Concat.input as Concat.input port + participant Concat.input2 as Concat.input port + participant Concat as Concat block + participant Concat.output as Concat.output port + participant BlockB as Another block + + BlockA-->>Concat: Connect + BlockA2-->>Concat: Connect + Concat-->>BlockB: Connect + + loop Concat process + BlockA->>Concat: Message + Concat->>Concat: Store message + BlockA2->>Concat: Message + Concat->>Concat: Store message + end + Concat->>Concat: Concat messages + Concat->>BlockB: Message + BlockA-->>Concat: Disconnect + BlockA2-->>Concat: Disconnect + Concat-->>Concat.input: Close + Concat-->>Concat.input2: Close + Concat-->>Concat.output: Close + Concat-->>BlockB: Disconnect diff --git a/lib/protoflow-blocks/doc/flow/replicate.mmd b/lib/protoflow-blocks/doc/flow/replicate.mmd new file mode 100644 index 00000000..e2c629ec --- /dev/null +++ b/lib/protoflow-blocks/doc/flow/replicate.mmd @@ -0,0 +1,17 @@ +block-beta + columns 7 + space:5 Sink1 space:1 + space:1 Source space:1 Replicate space:3 + space:5 Sink2 space:1 + + Source-- "input" -->Replicate + Replicate-- "output" -->Sink1 + Replicate-- "output" -->Sink2 + + classDef block height:48px,padding:8px; + classDef hidden visibility:none; + class Replicate block + class Source1 hidden + class Source2 hidden + class Sink1 hidden + class Sink2 hidden diff --git a/lib/protoflow-blocks/doc/flow/replicate.seq.mmd b/lib/protoflow-blocks/doc/flow/replicate.seq.mmd new file mode 100644 index 00000000..5966892b --- /dev/null +++ b/lib/protoflow-blocks/doc/flow/replicate.seq.mmd @@ -0,0 +1,27 @@ +sequenceDiagram + autonumber + participant BlockA as Another block + participant Replicate.input as Replicate.input port + participant Replicate as Replicate block + participant Replicate.output_1 as Replicate.output port + participant BlockB as Another block + participant Replicate.output_2 as Replicate.output port + participant BlockC as Another block + + BlockA-->>Replicate: Connect + Replicate-->>BlockB: Connect + Replicate-->>BlockC: Connect + + loop Replicate process + BlockA->>Replicate: Message + Replicate->>BlockB: Message + Replicate->>BlockC: Message + + end + + BlockA-->>Replicate: Disconnect + Replicate-->>Replicate.input: Close + Replicate-->>Replicate.output_1: Close + Replicate-->>BlockB: Disconnect + Replicate-->>Replicate.output_2: Close + Replicate-->>BlockC: Disconnect diff --git a/lib/protoflow-blocks/doc/flow/sort.mmd b/lib/protoflow-blocks/doc/flow/sort.mmd new file mode 100644 index 00000000..c182da61 --- /dev/null +++ b/lib/protoflow-blocks/doc/flow/sort.mmd @@ -0,0 +1,11 @@ +block-beta + columns 7 + Source space:2 Sort space:2 Sink + Source-- "input" -->Sort + Sort-- "output" -->Sink + + classDef block height:48px,padding:8px; + classDef hidden visibility:none; + class Sort block + class Source hidden + class Sink hidden diff --git a/lib/protoflow-blocks/doc/flow/sort.seq.mmd b/lib/protoflow-blocks/doc/flow/sort.seq.mmd new file mode 100644 index 00000000..7269070f --- /dev/null +++ b/lib/protoflow-blocks/doc/flow/sort.seq.mmd @@ -0,0 +1,22 @@ +sequenceDiagram + autonumber + participant BlockA as Another block + participant Sort.input as Sort.input port + participant Sort as Sort block + participant Sort.output as Sort.output port + participant BlockB as Another block + + BlockA-->>Sort: Connect + Sort-->>BlockB: Connect + + loop Sort process + BlockA->>Sort: Message + Sort->>Sort: Store message + end + + Sort->>Sort: Sort messages + Sort->>BlockB: Message + BlockA-->>Sort: Disconnect + Sort-->>Sort.input: Close + Sort-->>Sort.output: Close + Sort-->>BlockB: Disconnect diff --git a/lib/protoflow-blocks/src/blocks/flow/concat.rs b/lib/protoflow-blocks/src/blocks/flow/concat.rs index 8a119e2e..cc3fd4f6 100644 --- a/lib/protoflow-blocks/src/blocks/flow/concat.rs +++ b/lib/protoflow-blocks/src/blocks/flow/concat.rs @@ -1,23 +1,22 @@ // This is free and unencumbered software released into the public domain. extern crate std; -use core::ops::DerefMut; - use crate::prelude::{Arc, Vec}; use crate::{FlowBlocks, StdioConfig, StdioError, StdioSystem, SysBlocks, System}; use protoflow_core::{ types::Any, Block, BlockResult, BlockRuntime, InputPort, Message, OutputPort, }; +use protoflow_core::{BlockError, PortError}; use protoflow_derive::Block; use simple_mermaid::mermaid; -/// Divides a single input message stream into multiple output streams using a round-robin approach. +/// Combines multiple input streams into a single output stream in sequence. /// /// # Block Diagram -#[doc = mermaid!("../../../doc/flow/split.mmd")] +#[doc = mermaid!("../../../doc/flow/concat.mmd")] /// /// # Sequence Diagram -#[doc = mermaid!("../../../doc/flow/split.seq.mmd" framed)] +#[doc = mermaid!("../../../doc/flow/concat.seq.mmd" framed)] /// /// # Examples /// @@ -28,12 +27,16 @@ use simple_mermaid::mermaid; /// # fn main() { /// System::build(|s| { /// let stdin = s.read_stdin(); -/// let split = s.split(); -/// s.connect(&stdin.output, &split.input); +/// +/// let replicate = s.replicate(); +/// s.connect(&stdin.output, &replicate.input); +/// +/// let concat = s.block(Concat::new(s.input(), s.input(), s.output())); +/// s.connect(&replicate.output_1, &concat.input_1); +/// s.connect(&replicate.output_2, &concat.input_2); +/// /// let stdout_1 = s.write_stdout(); -/// s.connect(&split.output_1, &stdout_1.input); -/// let stdout_2 = s.write_stdout(); -/// s.connect(&split.output_2, &stdout_2.input); +/// s.connect(&concat.output, &stdout_1.input); /// }); /// # } /// ``` @@ -73,39 +76,74 @@ impl Concat { impl Block for Concat { fn execute(&mut self, runtime: &dyn BlockRuntime) -> BlockResult { + // Ensure the output channel is ready runtime.wait_for(&self.output)?; - let mut buffer1 = Vec::new(); - let mut buffer2 = Vec::new(); - let input1 = Arc::new(self.input_1.clone()); let input2 = Arc::new(self.input_2.clone()); - let input1_clone = Arc::clone(&input1); - let handle1 = std::thread::spawn(move || { - while let Ok(Some(message)) = input1_clone.recv() { - buffer1.push(message); + // Helper function to buffer messages from an input + fn buffer_input( + input: Arc>, + input_name: &str, + ) -> Result, PortError> { + let mut buffer = Vec::new(); + while let Ok(Some(message)) = input.recv() { + buffer.push(message); } - buffer1 - }); + tracing::info!("{} processed {} messages", input_name, buffer.len()); + Ok(buffer) + } - let input2_clone = Arc::clone(&input2); - let handle2 = std::thread::spawn(move || { - while let Ok(Some(message)) = input2_clone.recv() { - buffer2.push(message); - } - buffer2 + // Spawn threads to process and buffer messages from both inputs + let handle1 = std::thread::spawn({ + let input1 = Arc::clone(&input1); + move || buffer_input(input1, "input1") }); - let buffer1 = handle1.join().unwrap(); - let buffer2 = handle2.join().unwrap(); - - let mut combined = buffer1; - combined.extend(buffer2); + let handle2 = std::thread::spawn({ + let input2 = Arc::clone(&input2); + move || buffer_input(input2, "input2") + }); - for message in combined { - self.output.send(&message)?; + // Collect and handle thread results + let buffer1 = match handle1.join() { + Ok(result) => result.map_err(|e| { + tracing::error!("Error processing input1: {:?}", e); + BlockError::Other("Failed to process input1".into()) + })?, + Err(_) => { + tracing::error!("Thread for input1 panicked"); + return Err(BlockError::Other("Thread for input1 panicked".into())); + } + }; + + let buffer2 = match handle2.join() { + Ok(result) => result.map_err(|e| { + tracing::error!("Error processing input2: {:?}", e); + BlockError::Other("Failed to process input2".into()) + })?, + Err(_) => { + tracing::error!("Thread for input2 panicked"); + return Err(BlockError::Other("Thread for input2 panicked".into())); + } + }; + + // Concatenate and send messages to the output sequentially + tracing::info!( + "Concatenating {} messages from input1 with {} messages from input2", + buffer1.len(), + buffer2.len() + ); + + for message in buffer1.iter().chain(buffer2.iter()) { + if let Err(err) = self.output.send(message) { + tracing::error!("Failed to send message: {:?}", err); + return Err(err.into()); + } } + + tracing::info!("All messages successfully sent to the output."); Ok(()) } } @@ -119,13 +157,13 @@ impl StdioSystem for Concat { Ok(System::build(|s| { let stdin = s.read_stdin(); - let split = s.split(); - s.connect(&stdin.output, &split.input); + let replicate = s.replicate(); + s.connect(&stdin.output, &replicate.input); let concat = s.block(Concat::new(s.input(), s.input(), s.output())); - s.connect(&split.output_1, &concat.input_1); - s.connect(&split.output_2, &concat.input_2); + s.connect(&replicate.output_1, &concat.input_1); + s.connect(&replicate.output_2, &concat.input_2); let stdout_1 = s.write_stdout(); s.connect(&concat.output, &stdout_1.input); @@ -134,65 +172,17 @@ impl StdioSystem for Concat { } #[cfg(test)] -mod split_tests { - use crate::{Concat, CoreBlocks, FlowBlocks, StdioSystem, SysBlocks, System}; +mod concat_tests { + use crate::{FlowBlocks, System}; use protoflow_core::prelude::String; - use tracing::error; + extern crate std; #[test] fn instantiate_block() { // Check that the block is constructible: let _ = System::build(|s| { - let _ = s.split::(); + let _ = s.concat::(); }); } - - #[test] - #[ignore = "requires stdin"] - fn run_split_stdout_and_file() { - use super::*; - use protoflow_core::SystemBuilding; - if let Err(e) = System::run(|s| { - let stdin = s.read_stdin(); - let split = s.split(); - s.connect(&stdin.output, &split.input); - - let stdout_1 = s.write_stdout(); - s.connect(&split.output_1, &stdout_1.input); - - let file = s.const_string("text.txt"); - let write_file = s.write_file().with_flags(crate::WriteFlags { - create: true, - append: true, - }); - s.connect(&file.output, &write_file.path); - s.connect(&split.output_2, &write_file.input); - }) { - error!("{}", e) - } - } - - #[test] - #[ignore = "requires stdin"] - fn run_split_to_stdout() { - //use super::*; - use protoflow_core::SystemBuilding; - if let Err(e) = System::run(|s| { - let stdin = s.read_stdin(); - - let split = s.split(); - s.connect(&stdin.output, &split.input); - - let concat = s.block(Concat::new(s.input(), s.input(), s.output())); - - s.connect(&split.output_1, &concat.input_1); - s.connect(&split.output_2, &concat.input_2); - - let stdout_1 = s.write_stdout(); - s.connect(&concat.output, &stdout_1.input); - }) { - error!("{}", e) - } - } } diff --git a/lib/protoflow-blocks/src/blocks/flow/replicate.rs b/lib/protoflow-blocks/src/blocks/flow/replicate.rs index 870a15e1..251ded76 100644 --- a/lib/protoflow-blocks/src/blocks/flow/replicate.rs +++ b/lib/protoflow-blocks/src/blocks/flow/replicate.rs @@ -1,18 +1,19 @@ // This is free and unencumbered software released into the public domain. -use crate::{StdioConfig, StdioError, StdioSystem, SysBlocks, System}; +use crate::{FlowBlocks, StdioConfig, StdioError, StdioSystem, SysBlocks, System}; use protoflow_core::{ types::Any, Block, BlockResult, BlockRuntime, InputPort, Message, OutputPort, }; use protoflow_derive::Block; +use simple_mermaid::mermaid; -/// Divides a single input message stream into multiple output streams using a round-robin approach. +/// Duplicates a single input message stream into multiple identical output streams. /// /// # Block Diagram - +#[doc = mermaid!("../../../doc/flow/replicate.mmd")] /// /// # Sequence Diagram - +#[doc = mermaid!("../../../doc/flow/replicate.seq.mmd" framed)] /// /// # Examples /// @@ -23,10 +24,13 @@ use protoflow_derive::Block; /// # fn main() { /// System::build(|s| { /// let stdin = s.read_stdin(); +/// /// let replicate = s.replicate(); /// s.connect(&stdin.output, &replicate.input); +/// /// let stdout_1 = s.write_stdout(); /// s.connect(&replicate.output_1, &stdout_1.input); +/// /// let stdout_2 = s.write_stdout(); /// s.connect(&replicate.output_2, &stdout_2.input); /// }); @@ -89,7 +93,7 @@ impl StdioSystem for Replicate { Ok(System::build(|s| { let stdin = s.read_stdin(); - let replicate = s.block(Replicate::new(s.input(), s.output(), s.output())); + let replicate = s.replicate(); s.connect(&stdin.output, &replicate.input); let stdout_1 = s.write_stdout(); @@ -103,64 +107,16 @@ impl StdioSystem for Replicate { #[cfg(test)] mod replicate_tests { - use crate::{CoreBlocks, FlowBlocks, SysBlocks, System}; - use protoflow_core::{prelude::String, SystemBuilding}; - use tracing::error; + use crate::{FlowBlocks, System}; + use protoflow_core::prelude::String; - use super::Replicate; extern crate std; #[test] fn instantiate_block() { // Check that the block is constructible: let _ = System::build(|s| { - let _ = s.block(Replicate::new(s.input_any(), s.output(), s.output())); + let _ = s.replicate::(); }); } - - #[test] - #[ignore = "requires stdin"] - fn run_replicate_stdout_and_file() { - use super::*; - use protoflow_core::SystemBuilding; - if let Err(e) = System::run(|s| { - let stdin = s.read_stdin(); - let replicate = s.block(Replicate::new(s.input(), s.output_bytes(), s.output())); - s.connect(&stdin.output, &replicate.input); - - let stdout_1 = s.write_stdout(); - s.connect(&replicate.output_1, &stdout_1.input); - - let file = s.const_string("text.txt"); - let write_file = s.write_file().with_flags(crate::WriteFlags { - create: true, - append: true, - }); - s.connect(&file.output, &write_file.path); - s.connect(&replicate.output_2, &write_file.input); - }) { - error!("{}", e) - } - } - - #[test] - #[ignore = "requires stdin"] - fn run_replicate_to_stdout() { - //use super::*; - use protoflow_core::SystemBuilding; - if let Err(e) = System::run(|s| { - let stdin = s.read_stdin(); - - let replicate = s.block(Replicate::new(s.input(), s.output_bytes(), s.output())); - s.connect(&stdin.output, &replicate.input); - - let stdout_1 = s.write_stdout(); - s.connect(&replicate.output_1, &stdout_1.input); - - let stdout_2 = s.write_stdout(); - s.connect(&replicate.output_2, &stdout_2.input); - }) { - error!("{}", e) - } - } } diff --git a/lib/protoflow-blocks/src/blocks/flow/sort.rs b/lib/protoflow-blocks/src/blocks/flow/sort.rs index 51e859ec..6c0ac730 100644 --- a/lib/protoflow-blocks/src/blocks/flow/sort.rs +++ b/lib/protoflow-blocks/src/blocks/flow/sort.rs @@ -5,14 +5,15 @@ use protoflow_core::{ prelude::Vec, types::Any, Block, BlockResult, BlockRuntime, InputPort, Message, OutputPort, }; use protoflow_derive::Block; +use simple_mermaid::mermaid; -/// A block that simply stores all messages it receives. +/// Sorts a single input message stream in ascending order. /// /// # Block Diagram - +#[doc = mermaid!("../../../doc/flow/sort.mmd")] /// /// # Sequence Diagram - +#[doc = mermaid!("../../../doc/flow/concat.seq.mmd" framed)] /// /// # Examples /// diff --git a/lib/protoflow-blocks/src/blocks/flow/split.rs b/lib/protoflow-blocks/src/blocks/flow/split.rs index 66707036..9bfa4f68 100644 --- a/lib/protoflow-blocks/src/blocks/flow/split.rs +++ b/lib/protoflow-blocks/src/blocks/flow/split.rs @@ -45,10 +45,13 @@ pub struct Split { /// The input message stream. #[input] pub input: InputPort, + /// The output message stream #[output] pub output_1: OutputPort, + /// The output message stream #[output] pub output_2: OutputPort, + /// The internal state for keeping total number of messages #[state] pub message_count: u128, } @@ -150,7 +153,6 @@ mod split_tests { #[test] #[ignore = "requires stdin"] fn run_split_to_stdout() { - //use super::*; use protoflow_core::SystemBuilding; if let Err(e) = System::run(|s| { let stdin = s.read_stdin(); From c59b5d5849016d370ed4c497696f10b82d9fcd74 Mon Sep 17 00:00:00 2001 From: evren Date: Wed, 4 Dec 2024 09:32:27 +0300 Subject: [PATCH 06/24] merge flow --- README.md | 30 ++- lib/protoflow-blocks/doc/flow/merge.mmd | 15 ++ lib/protoflow-blocks/doc/flow/merge.seq.mmd | 26 +++ lib/protoflow-blocks/src/block_config.rs | 2 +- lib/protoflow-blocks/src/block_tag.rs | 4 + lib/protoflow-blocks/src/blocks/flow.rs | 19 ++ lib/protoflow-blocks/src/blocks/flow/merge.rs | 184 ++++++++++++++++++ lib/protoflow-blocks/src/lib.rs | 1 + lib/protoflow-blocks/src/system.rs | 8 +- 9 files changed, 285 insertions(+), 4 deletions(-) create mode 100644 lib/protoflow-blocks/doc/flow/merge.mmd create mode 100644 lib/protoflow-blocks/doc/flow/merge.seq.mmd create mode 100644 lib/protoflow-blocks/src/blocks/flow/merge.rs diff --git a/README.md b/README.md index 5e317314..2ff159f5 100644 --- a/README.md +++ b/README.md @@ -114,7 +114,7 @@ The built-in blocks provided by Protoflow are listed below: |:------------------|:-------------------------------------------------------------------------------------------------------------------------------| | [`Buffer`] | Stores all messages it receives. | | [`ConcatStrings`] | Concatenates the received string messages, with an optional delimiter string inserted between each message. | -| [`Concat`] | Merges multiple input message streams into a single output stream. | +| [`Concat`] | Merges multiple input message streams into a single output stream. | | [`Const`] | Sends a constant value. | | [`Count`] | Counts the number of messages it receives, while optionally passing them through. | | [`Decode`] | Decodes messages from a byte stream. | @@ -128,6 +128,7 @@ The built-in blocks provided by Protoflow are listed below: | [`EncodeHex`] | Encodes a byte stream into hexadecimal form. | | [`EncodeJSON`] | Encodes messages into JSON format. | | [`Hash`] | Computes the cryptographic hash of a byte stream. | +| [`Merge`] | Combines multiple input message streams into a single output stream by interleaving messages as they arrive. | | [`Random`] | Generates and sends a random value. | | [`ReadDir`] | Reads file names from a file system directory. | | [`ReadEnv`] | Reads the value of an environment variable. | @@ -487,6 +488,32 @@ block-beta protoflow execute Hash algorithm=blake3 ``` +#### [`Merge`] + +Combines multiple input message streams into a single output stream by interleaving messages as they arrive. + +```mermaid +block-beta + columns 7 + space:1 Source1 space:5 + space:3 Merge space:1 Sink space:1 + space:1 Source2 space:5 + Source1-- "input" -->Merge + Source2-- "input" -->Merge + Merge-- "output" -->Sink + + classDef block height:48px,padding:8px; + classDef hidden visibility:none; + class Merge block + class Source1 hidden + class Source2 hidden + class Sink hidden +``` + +```bash +protoflow execute Merge +``` + #### [`Random`] A block for generating and sending a random value. @@ -840,6 +867,7 @@ To add a new block type implementation, make sure to examine and amend: [`EncodeHex`]: https://docs.rs/protoflow-blocks/latest/protoflow_blocks/struct.EncodeHex.html [`EncodeJSON`]: https://docs.rs/protoflow-blocks/latest/protoflow_blocks/struct.EncodeJson.html [`Hash`]: https://docs.rs/protoflow-blocks/latest/protoflow_blocks/struct.Hash.html +[`Merge`]: https://docs.rs/protoflow-blocks/latest/protoflow_blocks/struct.Merge.html [`Random`]: https://docs.rs/protoflow-blocks/latest/protoflow_blocks/struct.Random.html [`ReadDir`]: https://docs.rs/protoflow-blocks/latest/protoflow_blocks/struct.ReadDir.html [`ReadEnv`]: https://docs.rs/protoflow-blocks/latest/protoflow_blocks/struct.ReadEnv.html diff --git a/lib/protoflow-blocks/doc/flow/merge.mmd b/lib/protoflow-blocks/doc/flow/merge.mmd new file mode 100644 index 00000000..90caf8e8 --- /dev/null +++ b/lib/protoflow-blocks/doc/flow/merge.mmd @@ -0,0 +1,15 @@ +block-beta + columns 7 + space:1 Source1 space:5 + space:3 Merge space:1 Sink space:1 + space:1 Source2 space:5 + Source1-- "input" -->Merge + Source2-- "input" -->Merge + Merge-- "output" -->Sink + + classDef block height:48px,padding:8px; + classDef hidden visibility:none; + class Merge block + class Source1 hidden + class Source2 hidden + class Sink hidden diff --git a/lib/protoflow-blocks/doc/flow/merge.seq.mmd b/lib/protoflow-blocks/doc/flow/merge.seq.mmd new file mode 100644 index 00000000..73d0f467 --- /dev/null +++ b/lib/protoflow-blocks/doc/flow/merge.seq.mmd @@ -0,0 +1,26 @@ +sequenceDiagram + autonumber + participant BlockA as Another block + participant BlockA2 as Another block + participant Merge.input as Merge.input port + participant Merge.input2 as Merge.input port + participant Merge as Merge block + participant Merge.output as Merge.output port + participant BlockB as Another block + + BlockA-->>Merge: Connect + BlockA2-->>Merge: Connect + Merge-->>BlockB: Connect + + loop Merge process + BlockA->>Merge: Message + Merge->>BlockB: Message + BlockA2->>Merge: Message + Merge->>BlockB: Message + end + BlockA-->>Merge: Disconnect + BlockA2-->>Merge: Disconnect + Merge-->>Merge.input: Close + Merge-->>Merge.input2: Close + Merge-->>Merge.output: Close + Merge-->>BlockB: Disconnect diff --git a/lib/protoflow-blocks/src/block_config.rs b/lib/protoflow-blocks/src/block_config.rs index fbe3ace3..c30598ba 100644 --- a/lib/protoflow-blocks/src/block_config.rs +++ b/lib/protoflow-blocks/src/block_config.rs @@ -57,7 +57,7 @@ impl<'de> serde::Deserialize<'de> for BlockConfig { .unwrap() } - "Concat" | "Replicate" | "Split" | "Sort" => { + "Concat" | "Merge" | "Replicate" | "Sort" | "Split" => { FlowBlockConfig::deserialize(value.clone()) .map(BlockConfig::Flow) .unwrap() diff --git a/lib/protoflow-blocks/src/block_tag.rs b/lib/protoflow-blocks/src/block_tag.rs index 9337983d..7c70e992 100644 --- a/lib/protoflow-blocks/src/block_tag.rs +++ b/lib/protoflow-blocks/src/block_tag.rs @@ -19,6 +19,7 @@ pub enum BlockTag { Random, // FlowBlocks Concat, + Merge, Replicate, Sort, Split, @@ -96,6 +97,7 @@ impl BlockTag { Encode => "Encode", EncodeHex => "EncodeHex", EncodeJson => "EncodeJSON", + Merge => "Merge", #[cfg(feature = "std")] ReadDir => "ReadDir", #[cfg(feature = "std")] @@ -151,6 +153,7 @@ impl FromStr for BlockTag { "Encode" => Encode, "EncodeHex" => EncodeHex, "EncodeJSON" => EncodeJson, + "Merge" => Merge, #[cfg(feature = "std")] "ReadDir" => ReadDir, #[cfg(feature = "std")] @@ -217,6 +220,7 @@ impl BlockInstantiation for BlockTag { Encode => Box::new(super::Encode::::with_system(system, None)), EncodeHex => Box::new(super::EncodeHex::with_system(system)), EncodeJson => Box::new(super::EncodeJson::with_system(system)), + Merge => Box::new(super::Merge::::with_system(system)), #[cfg(feature = "std")] ReadDir => Box::new(super::ReadDir::with_system(system)), #[cfg(feature = "std")] diff --git a/lib/protoflow-blocks/src/blocks/flow.rs b/lib/protoflow-blocks/src/blocks/flow.rs index c0ffea96..b235a2e5 100644 --- a/lib/protoflow-blocks/src/blocks/flow.rs +++ b/lib/protoflow-blocks/src/blocks/flow.rs @@ -12,6 +12,7 @@ pub mod flow { pub trait FlowBlocks { fn concat + 'static>(&mut self) -> Concat; + fn merge + 'static>(&mut self) -> Merge; fn replicate + 'static>(&mut self) -> Replicate; fn sort + PartialOrd + 'static>(&mut self) -> Sort; fn split + 'static>(&mut self) -> Split; @@ -21,6 +22,7 @@ pub mod flow { #[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] pub enum FlowBlockTag { Concat, + Merge, Replicate, Sort, Split, @@ -34,6 +36,11 @@ pub mod flow { input_2: InputPortName, output: OutputPortName, }, + Merge { + input_1: InputPortName, + input_2: InputPortName, + output: OutputPortName, + }, Replicate { input: InputPortName, output_1: OutputPortName, @@ -56,6 +63,7 @@ pub mod flow { use FlowBlockConfig::*; Cow::Borrowed(match self { Concat { .. } => "Concat", + Merge { .. } => "Merge", Replicate { .. } => "Replicate", Sort { .. } => "Sort", Split { .. } => "Split", @@ -70,6 +78,9 @@ pub mod flow { Concat { output, .. } => { vec![("output", Some(output.clone()))] } + Merge { output, .. } => { + vec![("output", Some(output.clone()))] + } Replicate { output_1, output_2, .. } => { @@ -103,6 +114,11 @@ pub mod flow { system.input_any(), system.output(), )), + Merge { .. } => Box::new(super::Merge::new( + system.input_any(), + system.input_any(), + system.output(), + )), Replicate { .. } => Box::new(super::Replicate::new( system.input_any(), system.output(), @@ -125,6 +141,9 @@ pub mod flow { mod concat; pub use concat::*; + mod merge; + pub use merge::*; + mod replicate; pub use replicate::*; diff --git a/lib/protoflow-blocks/src/blocks/flow/merge.rs b/lib/protoflow-blocks/src/blocks/flow/merge.rs new file mode 100644 index 00000000..ec0db67e --- /dev/null +++ b/lib/protoflow-blocks/src/blocks/flow/merge.rs @@ -0,0 +1,184 @@ +// This is free and unencumbered software released into the public domain. +extern crate std; + +use crate::prelude::{format, Arc}; +use crate::{FlowBlocks, StdioConfig, StdioError, StdioSystem, SysBlocks, System}; +use protoflow_core::BlockError; +use protoflow_core::{ + types::Any, Block, BlockResult, BlockRuntime, InputPort, Message, OutputPort, +}; +use protoflow_derive::Block; +use simple_mermaid::mermaid; + +/// Combines multiple input message streams into a single output stream by interleaving messages as they arrive. +/// +/// # Block Diagram +#[doc = mermaid!("../../../doc/flow/merge.mmd")] +/// +/// # Sequence Diagram +#[doc = mermaid!("../../../doc/flow/merge.seq.mmd" framed)] +/// +/// # Examples +/// +/// ## Using the block in a system +/// +/// ```rust +/// # use protoflow_blocks::*; +/// # fn main() { +/// System::build(|s| { +/// let stdin = s.read_stdin(); +/// +/// let replicate = s.replicate(); +/// s.connect(&stdin.output, &replicate.input); +/// +/// let merge = s.block(Merge::new(s.input(), s.input(), s.output())); +/// s.connect(&replicate.output_1, &merge.input_1); +/// s.connect(&replicate.output_2, &merge.input_2); +/// +/// let stdout_1 = s.write_stdout(); +/// s.connect(&merge.output, &stdout_1.input); +/// }); +/// # } +/// ``` +/// +/// ## Running the block via the CLI +/// +/// ```console +/// $ protoflow execute Merge +/// ``` +/// +#[derive(Block, Clone)] +pub struct Merge { + /// The input message stream. + #[input] + pub input_1: InputPort, + #[output] + pub input_2: InputPort, + #[output] + pub output: OutputPort, +} + +impl Merge { + pub fn new(input_1: InputPort, input_2: InputPort, output: OutputPort) -> Self { + Self { + input_1, + input_2, + output, + } + } +} +impl Merge { + pub fn with_system(system: &System) -> Self { + use crate::SystemBuilding; + Self::new(system.input(), system.input(), system.output()) + } +} + +impl Block for Merge { + fn execute(&mut self, runtime: &dyn BlockRuntime) -> BlockResult { + runtime.wait_for(&self.output)?; + + let input1 = Arc::new(self.input_1.clone()); + let input2 = Arc::new(self.input_2.clone()); + let output = Arc::new(self.output.clone()); + + fn process_port( + input: Arc>, + output: Arc>, + ) -> Result<(), BlockError> { + while let Ok(Some(message)) = input.recv() { + if let Err(err) = output.send(&message) { + tracing::error!("Error sending message: {}", err); + return Err(BlockError::Other(format!("Error sending message: {}", err))); + } + } + Ok(()) + } + + let input1_thread = { + let input1_clone = Arc::clone(&input1); + let output_clone = Arc::clone(&output); + std::thread::spawn(move || process_port(input1_clone, output_clone)) + }; + + let input2_thread = { + let input2_clone = Arc::clone(&input2); + let output_clone = Arc::clone(&output); + std::thread::spawn(move || process_port(input2_clone, output_clone)) + }; + + if let Err(_) = input1_thread.join() { + tracing::error!("Thread for input1 panicked"); + return Err(BlockError::Other("Thread for input1 panicked".into())); + } + + if let Err(_) = input2_thread.join() { + tracing::error!("Thread for input2 panicked"); + return Err(BlockError::Other("Thread for input2 panicked".into())); + } + + Ok(()) + } +} + +#[cfg(feature = "std")] +impl StdioSystem for Merge { + fn build_system(config: StdioConfig) -> Result { + use crate::SystemBuilding; + config.reject_any()?; + + Ok(System::build(|s| { + let stdin = s.read_stdin(); + + let replicate = s.replicate(); + s.connect(&stdin.output, &replicate.input); + + let merge = s.block(Merge::new(s.input(), s.input(), s.output())); + + s.connect(&replicate.output_1, &merge.input_1); + s.connect(&replicate.output_2, &merge.input_2); + + let stdout_1 = s.write_stdout(); + s.connect(&merge.output, &stdout_1.input); + })) + } +} + +#[cfg(test)] +mod merge_tests { + use crate::{FlowBlocks, SysBlocks, System}; + use protoflow_core::{prelude::String, SystemBuilding}; + use tracing::error; + + use super::Merge; + + extern crate std; + + #[test] + fn instantiate_block() { + // Check that the block is constructible: + let _ = System::build(|s| { + let _ = s.merge::(); + }); + } + #[test] + #[ignore = "requires stdin"] + fn run_block() { + if let Err(e) = System::run(|s| { + let stdin = s.read_stdin(); + + let replicate = s.replicate(); + s.connect(&stdin.output, &replicate.input); + + let merge = s.block(Merge::new(s.input(), s.input(), s.output())); + + s.connect(&replicate.output_1, &merge.input_1); + s.connect(&replicate.output_2, &merge.input_2); + + let stdout_1 = s.write_stdout(); + s.connect(&merge.output, &stdout_1.input); + }) { + error!("{}", e) + } + } +} diff --git a/lib/protoflow-blocks/src/lib.rs b/lib/protoflow-blocks/src/lib.rs index 0356b5fa..ca1f36d7 100644 --- a/lib/protoflow-blocks/src/lib.rs +++ b/lib/protoflow-blocks/src/lib.rs @@ -61,6 +61,7 @@ pub fn build_stdio_system( "Random" => Random::::build_system(config)?, // FlowBlocks "Concat" => Concat::::build_system(config)?, + "Merge" => Merge::::build_system(config)?, "Replicate" => Replicate::::build_system(config)?, "Sort" => Sort::::build_system(config)?, "Split" => Split::::build_system(config)?, diff --git a/lib/protoflow-blocks/src/system.rs b/lib/protoflow-blocks/src/system.rs index bba614ba..85a9aa96 100644 --- a/lib/protoflow-blocks/src/system.rs +++ b/lib/protoflow-blocks/src/system.rs @@ -7,8 +7,8 @@ use crate::{ types::{DelayType, Encoding}, AllBlocks, Buffer, Concat, ConcatStrings, Const, CoreBlocks, Count, Decode, DecodeCsv, DecodeHex, DecodeJson, Delay, Drop, Encode, EncodeCsv, EncodeHex, EncodeJson, FlowBlocks, - HashBlocks, IoBlocks, MathBlocks, Random, ReadDir, ReadEnv, ReadFile, ReadSocket, ReadStdin, - Replicate, Sort, Split, SplitString, SysBlocks, TextBlocks, WriteFile, WriteSocket, + HashBlocks, IoBlocks, MathBlocks, Merge, Random, ReadDir, ReadEnv, ReadFile, ReadSocket, + ReadStdin, Replicate, Sort, Split, SplitString, SysBlocks, TextBlocks, WriteFile, WriteSocket, WriteStderr, WriteStdout, }; use protoflow_core::{ @@ -169,6 +169,10 @@ impl FlowBlocks for System { self.0.block(Concat::::with_system(self)) } + fn merge + 'static>(&mut self) -> Merge { + self.0.block(Merge::::with_system(self)) + } + fn replicate + 'static>(&mut self) -> Replicate { self.0.block(Replicate::::with_system(self)) } From 6d18ff4174c3e8e5d1769fd865e1869c8f14d430 Mon Sep 17 00:00:00 2001 From: evren Date: Wed, 4 Dec 2024 14:55:50 +0300 Subject: [PATCH 07/24] comparable any --- lib/protoflow-blocks/src/block_tag.rs | 4 +- lib/protoflow-blocks/src/blocks/flow.rs | 5 +- lib/protoflow-blocks/src/blocks/flow/sort.rs | 73 ++++++++-------- lib/protoflow-core/src/comparable_any.rs | 87 ++++++++++++++++++++ lib/protoflow-core/src/lib.rs | 3 + 5 files changed, 129 insertions(+), 43 deletions(-) create mode 100644 lib/protoflow-core/src/comparable_any.rs diff --git a/lib/protoflow-blocks/src/block_tag.rs b/lib/protoflow-blocks/src/block_tag.rs index 9337983d..68f3fae2 100644 --- a/lib/protoflow-blocks/src/block_tag.rs +++ b/lib/protoflow-blocks/src/block_tag.rs @@ -5,7 +5,7 @@ use crate::{ BlockInstantiation, System, }; use enum_iterator::Sequence; -use protoflow_core::{types::Any, Block}; +use protoflow_core::{types::Any, Block, ComparableAny}; #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] #[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd, Sequence)] @@ -228,7 +228,7 @@ impl BlockInstantiation for BlockTag { #[cfg(feature = "std")] ReadStdin => Box::new(super::ReadStdin::with_system(system, None)), Replicate => Box::new(super::Replicate::::with_system(system)), - Sort => Box::new(super::Sort::::with_system(system)), + Sort => Box::new(super::Sort::::with_system(system)), Split => Box::new(super::Split::::with_system(system)), #[cfg(feature = "std")] WriteFile => Box::new(super::WriteFile::with_system(system, None)), diff --git a/lib/protoflow-blocks/src/blocks/flow.rs b/lib/protoflow-blocks/src/blocks/flow.rs index c0ffea96..446f1d0e 100644 --- a/lib/protoflow-blocks/src/blocks/flow.rs +++ b/lib/protoflow-blocks/src/blocks/flow.rs @@ -8,7 +8,7 @@ pub mod flow { BlockConnections, BlockInstantiation, System, }; - use protoflow_core::{Block, Message}; + use protoflow_core::{Block, ComparableAny, Message}; pub trait FlowBlocks { fn concat + 'static>(&mut self) -> Concat; @@ -109,8 +109,7 @@ pub mod flow { system.output(), )), Sort { .. } => Box::new(super::Sort::new( - system.input_any(), - system.input(), + system.input::(), system.output(), )), Split { .. } => Box::new(super::Split::new( diff --git a/lib/protoflow-blocks/src/blocks/flow/sort.rs b/lib/protoflow-blocks/src/blocks/flow/sort.rs index 6c0ac730..cdf33747 100644 --- a/lib/protoflow-blocks/src/blocks/flow/sort.rs +++ b/lib/protoflow-blocks/src/blocks/flow/sort.rs @@ -1,11 +1,14 @@ // This is free and unencumbered software released into the public domain. +use core::cmp::Ordering; + use crate::{StdioConfig, StdioError, StdioSystem, System}; use protoflow_core::{ prelude::Vec, types::Any, Block, BlockResult, BlockRuntime, InputPort, Message, OutputPort, }; use protoflow_derive::Block; use simple_mermaid::mermaid; +use tracing::error; /// Sorts a single input message stream in ascending order. /// @@ -13,7 +16,7 @@ use simple_mermaid::mermaid; #[doc = mermaid!("../../../doc/flow/sort.mmd")] /// /// # Sequence Diagram -#[doc = mermaid!("../../../doc/flow/concat.seq.mmd" framed)] +#[doc = mermaid!("../../../doc/flow/sort.seq.mmd" framed)] /// /// # Examples /// @@ -42,9 +45,6 @@ pub struct Sort { #[input] pub input: InputPort, - #[input] - pub stop: InputPort, - /// The output message stream. #[output] pub output: OutputPort, @@ -55,10 +55,9 @@ pub struct Sort { } impl Sort { - pub fn new(input: InputPort, stop: InputPort, output: OutputPort) -> Self { + pub fn new(input: InputPort, output: OutputPort) -> Self { Self { input, - stop, output, messages: Vec::new(), } @@ -72,25 +71,31 @@ impl Sort { impl Sort { pub fn with_system(system: &System) -> Self { use crate::SystemBuilding; - Self::new(system.input(), system.input(), system.output()) + Self::new(system.input(), system.output()) } } -impl Block for Sort { +impl Block for Sort { fn execute(&mut self, _runtime: &dyn BlockRuntime) -> BlockResult { - // if let Some(x) = self.stop.recv()? { - // while let Some(message) = self.input.recv()? { - // if message == x { - // self.messages.sort_by(|x, y| x.partial_cmp(y).unwrap()); - // for x in self.messages.iter() { - // self.output.send(x)?; - // } - // self.messages.clear(); - // } else { - // self.messages.push(message); - // } - // } - // } + while let Some(message) = self.input.recv()? { + self.messages.push(message); + } + + self.messages.sort_by(|x, y| { + if let Some(ordering) = x.partial_cmp(y) { + ordering + } else { + error!("Incomparable values: {:?} and {:?}", x, y); + Ordering::Equal + } + }); + + for x in self.messages.iter() { + self.output.send(x)?; + } + + self.messages.clear(); + Ok(()) } } @@ -104,7 +109,7 @@ impl StdioSystem for Sort { Ok(System::build(|s| { let stdin = config.read_stdin(s); - let sort = s.block(Sort::new(s.input(), s.input(), s.output())); + let sort = s.block(Sort::new(s.input(), s.output())); s.connect(&stdin.output, &sort.input); })) } @@ -119,33 +124,25 @@ mod tests { fn instantiate_block() { // Check that the block is constructible: let _ = System::build(|s| { - let _ = s.block(Sort::::new(s.input(), s.input(), s.output())); + let _ = s.block(Sort::::new(s.input(), s.output())); }); } -} - -#[cfg(test)] -mod split_tests { - use bytes::Bytes; - use tracing::error; - - use crate::{Const, SysBlocks}; #[test] #[ignore = "requires stdin"] - fn run_split_stdout_and_file() { + fn run_sort_stdout() { use super::*; + use crate::SysBlocks; use protoflow_core::SystemBuilding; + use tracing::error; + if let Err(e) = System::run(|s| { let stdin = s.read_stdin(); - let split = s.block(Sort::new(s.input(), s.input(), s.output())); - s.connect(&stdin.output, &split.input); - - let constant = s.block(Const::::with_params(s.output(), Bytes::from("\n"))); - s.connect(&constant.output, &split.stop); + let sort = s.block(Sort::new(s.input(), s.output())); + s.connect(&stdin.output, &sort.input); let stdout_1 = s.write_stdout(); - s.connect(&split.output, &stdout_1.input); + s.connect(&sort.output, &stdout_1.input); }) { error!("{}", e) } diff --git a/lib/protoflow-core/src/comparable_any.rs b/lib/protoflow-core/src/comparable_any.rs new file mode 100644 index 00000000..52c8f55b --- /dev/null +++ b/lib/protoflow-core/src/comparable_any.rs @@ -0,0 +1,87 @@ +// This is free and unencumbered software released into the public domain. + +use core::{ + cmp::Ordering, + ops::{Deref, DerefMut}, +}; + +use crate::{prelude::prost_types::Any, Message}; + +#[derive(Debug, Clone, Default)] +pub struct ComparableAny(pub Any); + +impl PartialOrd for ComparableAny { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.0.type_url.cmp(&other.0.type_url)) + } +} + +impl PartialEq for ComparableAny { + fn eq(&self, other: &Self) -> bool { + self.0.type_url == other.0.type_url + } +} + +impl From for ComparableAny { + fn from(any: Any) -> Self { + ComparableAny(any) + } +} + +impl From for Any { + fn from(comparable: ComparableAny) -> Self { + comparable.0 + } +} + +impl AsRef for ComparableAny { + fn as_ref(&self) -> &Any { + &self.0 + } +} + +impl AsMut for ComparableAny { + fn as_mut(&mut self) -> &mut Any { + &mut self.0 + } +} + +impl Deref for ComparableAny { + type Target = Any; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl DerefMut for ComparableAny { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.0 + } +} + +impl prost::Message for ComparableAny { + fn encode_raw(&self, buf: &mut impl bytes::BufMut) { + self.0.encode_raw(buf); + } + + fn merge_field( + &mut self, + tag: u32, + wire_type: prost::encoding::WireType, + buf: &mut impl bytes::Buf, + ctx: prost::encoding::DecodeContext, + ) -> Result<(), prost::DecodeError> { + self.0.merge_field(tag, wire_type, buf, ctx) + } + + fn encoded_len(&self) -> usize { + self.0.encoded_len() + } + + fn clear(&mut self) { + self.0.clear(); + } +} + +impl Message for ComparableAny {} diff --git a/lib/protoflow-core/src/lib.rs b/lib/protoflow-core/src/lib.rs index d82daaf3..40f4ccf2 100644 --- a/lib/protoflow-core/src/lib.rs +++ b/lib/protoflow-core/src/lib.rs @@ -25,6 +25,9 @@ pub use block_error::*; mod block_runtime; pub use block_runtime::*; +mod comparable_any; +pub use comparable_any::*; + mod function_block; pub use function_block::*; From 9f1e8fd0945815a1e35ccaac1fa439b952d0b561 Mon Sep 17 00:00:00 2001 From: evren Date: Mon, 9 Dec 2024 13:41:44 +0300 Subject: [PATCH 08/24] fix readme --- README.md | 77 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 77 insertions(+) diff --git a/README.md b/README.md index 2ff159f5..939a26c2 100644 --- a/README.md +++ b/README.md @@ -164,6 +164,32 @@ block-beta protoflow execute Buffer ``` +#### [`Concat`] + +Concatenates multiple input message streams into a single output stream. + +```mermaid +block-beta + columns 7 + space:1 Source1 space:5 + space:3 Concat space:1 Sink space:1 + space:1 Source2 space:5 + Source1-- "input" -->Concat + Source2-- "input" -->Concat + Concat-- "output" -->Sink + + classDef block height:48px,padding:8px; + classDef hidden visibility:none; + class Concat block + class Source1 hidden + class Source2 hidden + class Sink hidden +``` + +```bash +protoflow execute Concat +``` + #### [`ConcatStrings`] A block for concatenating all string messages it receives, with an optional delimiter string inserted between each message @@ -649,6 +675,57 @@ block-beta protoflow execute ReadStdin < input.txt ``` +#### [`Replicate`] + +Duplicates a single input message stream into multiple identical output streams. + +```mermaid +block-beta + columns 7 + space:5 Sink1 space:1 + space:1 Source space:1 Replicate space:3 + space:5 Sink2 space:1 + + Source-- "input" -->Replicate + Replicate-- "output" -->Sink1 + Replicate-- "output" -->Sink2 + + classDef block height:48px,padding:8px; + classDef hidden visibility:none; + class Replicate block + class Source1 hidden + class Source2 hidden + class Sink1 hidden + class Sink2 hidden +``` + +```bash +protoflow execute Replicate +``` + +#### [`Sort`] + +Sorts a single input message stream in ascending order. + +```mermaid +block-beta + columns 7 + Source space:2 Sort space:2 Sink + Source-- "input" -->Sort + Sort-- "output" -->Sink + + classDef block height:48px,padding:8px; + classDef hidden visibility:none; + class Sort block + class Source hidden + class Sink hidden + class Sink2 hidden +``` + +```bash +protoflow execute Sort +``` + #### [`Split`] Divides a single input message stream into multiple output streams using a round-robin approach. From 4109572cde9ba65e14a7c5715817a110a9e77f76 Mon Sep 17 00:00:00 2001 From: evren Date: Mon, 9 Dec 2024 14:00:56 +0300 Subject: [PATCH 09/24] remove stop port from sort block --- lib/protoflow-blocks/src/blocks/flow.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/protoflow-blocks/src/blocks/flow.rs b/lib/protoflow-blocks/src/blocks/flow.rs index 8aec73c9..692052d5 100644 --- a/lib/protoflow-blocks/src/blocks/flow.rs +++ b/lib/protoflow-blocks/src/blocks/flow.rs @@ -48,7 +48,6 @@ pub mod flow { }, Sort { input: InputPortName, - stop: InputPortName, output: OutputPortName, }, Split { From 2059ef515babad67c8f5a594b0693c561ff16b1f Mon Sep 17 00:00:00 2001 From: evren Date: Mon, 9 Dec 2024 16:56:10 +0300 Subject: [PATCH 10/24] distinct --- README.md | 28 +++- lib/protoflow-blocks/doc/flow/distinct.mmd | 11 ++ .../doc/flow/distinct.seq.mmd | 21 +++ lib/protoflow-blocks/src/block_config.rs | 2 +- lib/protoflow-blocks/src/block_tag.rs | 4 + lib/protoflow-blocks/src/blocks/flow.rs | 17 ++- .../src/blocks/flow/distinct.rs | 140 ++++++++++++++++++ lib/protoflow-blocks/src/lib.rs | 1 + lib/protoflow-blocks/src/system.rs | 12 +- 9 files changed, 228 insertions(+), 8 deletions(-) create mode 100644 lib/protoflow-blocks/doc/flow/distinct.mmd create mode 100644 lib/protoflow-blocks/doc/flow/distinct.seq.mmd create mode 100644 lib/protoflow-blocks/src/blocks/flow/distinct.rs diff --git a/README.md b/README.md index 2ff159f5..715196c4 100644 --- a/README.md +++ b/README.md @@ -114,7 +114,7 @@ The built-in blocks provided by Protoflow are listed below: |:------------------|:-------------------------------------------------------------------------------------------------------------------------------| | [`Buffer`] | Stores all messages it receives. | | [`ConcatStrings`] | Concatenates the received string messages, with an optional delimiter string inserted between each message. | -| [`Concat`] | Merges multiple input message streams into a single output stream. | +| [`Concat`] | Concatenates multiple input message streams into a single output stream. | | [`Const`] | Sends a constant value. | | [`Count`] | Counts the number of messages it receives, while optionally passing them through. | | [`Decode`] | Decodes messages from a byte stream. | @@ -122,13 +122,14 @@ The built-in blocks provided by Protoflow are listed below: | [`DecodeHex`] | Decodes hexadecimal stream to byte stream. | | [`DecodeJSON`] | Decodes JSON messages from a byte stream. | | [`Delay`] | Passes messages through while delaying them by a fixed or random duration. | +| [`Distinct`] | Removes duplicate values from the input stream. | | [`Drop`] | Discards all messages it receives. | | [`Encode`] | Encodes messages to a byte stream. | | [`EncodeCSV`] | Encodes the provided header and rows, given as `prost_types::Value`, into a CSV-formatted byte stream. | | [`EncodeHex`] | Encodes a byte stream into hexadecimal form. | | [`EncodeJSON`] | Encodes messages into JSON format. | | [`Hash`] | Computes the cryptographic hash of a byte stream. | -| [`Merge`] | Combines multiple input message streams into a single output stream by interleaving messages as they arrive. | +| [`Merge`] | Merges multiple input message streams into a single output stream by interleaving messages as they arrive. | | [`Random`] | Generates and sends a random value. | | [`ReadDir`] | Reads file names from a file system directory. | | [`ReadEnv`] | Reads the value of an environment variable. | @@ -255,6 +256,28 @@ block-beta protoflow execute Decode encoding=text ``` +#### [`Distinct`] + +Removes duplicate values from the input stream. + +```mermaid +block-beta + columns 7 + Source space:2 Distinct space:2 Sink + Source-- "input" -->Distinct + Distinct-- "output" -->Sink + + classDef block height:48px,padding:8px; + classDef hidden visibility:none; + class Distinct block + class Source hidden + class Sink hidden +``` + +```bash +protoflow execute Distinct +``` + #### [`DecodeCSV`] A block that decodes CSV files from a byte stream into a header and rows represented as `prost_types::Value` @@ -861,6 +884,7 @@ To add a new block type implementation, make sure to examine and amend: [`DecodeHex`]: https://docs.rs/protoflow-blocks/latest/protoflow_blocks/struct.DecodeHex.html [`DecodeJSON`]: https://docs.rs/protoflow-blocks/latest/protoflow_blocks/struct.DecodeJson.html [`Delay`]: https://docs.rs/protoflow-blocks/latest/protoflow_blocks/struct.Delay.html +[`Distinct`]: https://docs.rs/protoflow-blocks/latest/protoflow_blocks/struct.Distinct.html [`Drop`]: https://docs.rs/protoflow-blocks/latest/protoflow_blocks/struct.Drop.html [`Encode`]: https://docs.rs/protoflow-blocks/latest/protoflow_blocks/struct.Encode.html [`EncodeCSV`]: https://docs.rs/protoflow-blocks/latest/protoflow_blocks/struct.EncodeCsv.html diff --git a/lib/protoflow-blocks/doc/flow/distinct.mmd b/lib/protoflow-blocks/doc/flow/distinct.mmd new file mode 100644 index 00000000..a9468026 --- /dev/null +++ b/lib/protoflow-blocks/doc/flow/distinct.mmd @@ -0,0 +1,11 @@ +block-beta + columns 7 + Source space:2 Distinct space:2 Sink + Source-- "input" -->Distinct + Distinct-- "output" -->Sink + + classDef block height:48px,padding:8px; + classDef hidden visibility:none; + class Distinct block + class Source hidden + class Sink hidden diff --git a/lib/protoflow-blocks/doc/flow/distinct.seq.mmd b/lib/protoflow-blocks/doc/flow/distinct.seq.mmd new file mode 100644 index 00000000..c8a33715 --- /dev/null +++ b/lib/protoflow-blocks/doc/flow/distinct.seq.mmd @@ -0,0 +1,21 @@ +sequenceDiagram + autonumber + participant BlockA as Another block + participant Distinct.input as Distinct.input port + participant Distinct as Distinct block + participant Distinct.output as Distinct.output port + participant BlockB as Another block + + BlockA-->>Distinct: Connect + Distinct-->>BlockB: Connect + + loop Distinct process + BlockA->>Distinct: Message + Distinct->>Distinct: Store distinct messages + end + + Distinct->>BlockB: Message + BlockA-->>Distinct: Disconnect + Distinct-->>Distinct.input: Close + Distinct-->>Distinct.output: Close + Distinct-->>BlockB: Disconnect diff --git a/lib/protoflow-blocks/src/block_config.rs b/lib/protoflow-blocks/src/block_config.rs index c30598ba..fffb27d4 100644 --- a/lib/protoflow-blocks/src/block_config.rs +++ b/lib/protoflow-blocks/src/block_config.rs @@ -57,7 +57,7 @@ impl<'de> serde::Deserialize<'de> for BlockConfig { .unwrap() } - "Concat" | "Merge" | "Replicate" | "Sort" | "Split" => { + "Concat" | "Distinct" | "Merge" | "Replicate" | "Sort" | "Split" => { FlowBlockConfig::deserialize(value.clone()) .map(BlockConfig::Flow) .unwrap() diff --git a/lib/protoflow-blocks/src/block_tag.rs b/lib/protoflow-blocks/src/block_tag.rs index 12d48a09..0b627649 100644 --- a/lib/protoflow-blocks/src/block_tag.rs +++ b/lib/protoflow-blocks/src/block_tag.rs @@ -19,6 +19,7 @@ pub enum BlockTag { Random, // FlowBlocks Concat, + Distinct, Merge, Replicate, Sort, @@ -94,6 +95,7 @@ impl BlockTag { Decode => "Decode", DecodeHex => "DecodeHex", DecodeJson => "DecodeJSON", + Distinct => "Distinct", Encode => "Encode", EncodeHex => "EncodeHex", EncodeJson => "EncodeJSON", @@ -138,6 +140,7 @@ impl FromStr for BlockTag { "Const" => Const, "Count" => Count, "Delay" => Delay, + "Distinct" => Distinct, "Drop" => Drop, "Random" => Random, #[cfg(any( @@ -217,6 +220,7 @@ impl BlockInstantiation for BlockTag { Decode => Box::new(super::Decode::::with_system(system, None)), DecodeHex => Box::new(super::DecodeHex::with_system(system)), DecodeJson => Box::new(super::DecodeJson::with_system(system)), + Distinct => Box::new(super::Distinct::::with_system(system)), Encode => Box::new(super::Encode::::with_system(system, None)), EncodeHex => Box::new(super::EncodeHex::with_system(system)), EncodeJson => Box::new(super::EncodeJson::with_system(system)), diff --git a/lib/protoflow-blocks/src/blocks/flow.rs b/lib/protoflow-blocks/src/blocks/flow.rs index 8aec73c9..3a03056d 100644 --- a/lib/protoflow-blocks/src/blocks/flow.rs +++ b/lib/protoflow-blocks/src/blocks/flow.rs @@ -12,6 +12,7 @@ pub mod flow { pub trait FlowBlocks { fn concat + 'static>(&mut self) -> Concat; + fn distinct + PartialEq + 'static>(&mut self) -> Distinct; fn merge + 'static>(&mut self) -> Merge; fn replicate + 'static>(&mut self) -> Replicate; fn sort + PartialOrd + 'static>(&mut self) -> Sort; @@ -22,6 +23,7 @@ pub mod flow { #[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] pub enum FlowBlockTag { Concat, + Distinct, Merge, Replicate, Sort, @@ -36,6 +38,10 @@ pub mod flow { input_2: InputPortName, output: OutputPortName, }, + Distinct { + input: InputPortName, + output: OutputPortName, + }, Merge { input_1: InputPortName, input_2: InputPortName, @@ -48,7 +54,6 @@ pub mod flow { }, Sort { input: InputPortName, - stop: InputPortName, output: OutputPortName, }, Split { @@ -63,6 +68,7 @@ pub mod flow { use FlowBlockConfig::*; Cow::Borrowed(match self { Concat { .. } => "Concat", + Distinct { .. } => "Distinct", Merge { .. } => "Merge", Replicate { .. } => "Replicate", Sort { .. } => "Sort", @@ -78,6 +84,9 @@ pub mod flow { Concat { output, .. } => { vec![("output", Some(output.clone()))] } + Distinct { output, .. } => { + vec![("output", Some(output.clone()))] + } Merge { output, .. } => { vec![("output", Some(output.clone()))] } @@ -114,6 +123,9 @@ pub mod flow { system.input_any(), system.output(), )), + Distinct { .. } => { + Box::new(super::Distinct::new(system.input_any(), system.output())) + } Merge { .. } => Box::new(super::Merge::new( system.input_any(), system.input_any(), @@ -140,6 +152,9 @@ pub mod flow { mod concat; pub use concat::*; + mod distinct; + pub use distinct::*; + mod merge; pub use merge::*; diff --git a/lib/protoflow-blocks/src/blocks/flow/distinct.rs b/lib/protoflow-blocks/src/blocks/flow/distinct.rs new file mode 100644 index 00000000..d8571800 --- /dev/null +++ b/lib/protoflow-blocks/src/blocks/flow/distinct.rs @@ -0,0 +1,140 @@ +// This is free and unencumbered software released into the public domain. + +use crate::{StdioConfig, StdioError, StdioSystem, System}; +use protoflow_core::{ + prelude::Vec, types::Any, Block, BlockResult, BlockRuntime, InputPort, Message, OutputPort, +}; +use protoflow_derive::Block; +use simple_mermaid::mermaid; + +/// Removes duplicate values from the input stream. +/// +/// # Block Diagram +#[doc = mermaid!("../../../doc/flow/distinct.mmd")] +/// +/// # Sequence Diagram +#[doc = mermaid!("../../../doc/flow/distinct.seq.mmd" framed)] +/// +/// # Examples +/// +/// ## Using the block in a system +/// +/// ```rust +/// # use protoflow_blocks::*; +/// # fn main() { +/// System::build(|s| { +/// let stdin = s.read_stdin(); +/// let distinct = s.distinct(); +/// s.connect(&stdin.output, &distinct.input); +/// }); +/// # } +/// ``` +/// +/// ## Running the block via the CLI +/// +/// ```console +/// $ protoflow execute Distinct +/// ``` +/// +#[derive(Block, Clone)] +pub struct Distinct { + /// The input message stream. + #[input] + pub input: InputPort, + + /// The output message stream. + #[output] + pub output: OutputPort, + + /// The internal state storing the messages received. + #[state] + messages: Vec, +} + +impl Distinct { + pub fn new(input: InputPort, output: OutputPort) -> Self { + Self { + input, + output, + messages: Vec::new(), + } + } + + pub fn messages(&self) -> &Vec { + &self.messages + } +} + +impl Distinct { + pub fn with_system(system: &System) -> Self { + use crate::SystemBuilding; + Self::new(system.input(), system.output()) + } +} + +impl Block for Distinct { + fn execute(&mut self, _runtime: &dyn BlockRuntime) -> BlockResult { + while let Some(message) = self.input.recv()? { + if !self.messages.contains(&message) { + self.messages.push(message); + } + } + + for message in self.messages.iter() { + self.output.send(message)?; + } + + self.messages.clear(); + + Ok(()) + } +} + +#[cfg(feature = "std")] +impl StdioSystem for Distinct { + fn build_system(config: StdioConfig) -> Result { + use crate::SystemBuilding; + + config.reject_any()?; + + Ok(System::build(|s| { + let stdin = config.read_stdin(s); + let distinct = s.block(Distinct::new(s.input(), s.output())); + s.connect(&stdin.output, &distinct.input); + })) + } +} + +#[cfg(test)] +mod tests { + use super::Distinct; + use crate::{System, SystemBuilding}; + + #[test] + fn instantiate_block() { + // Check that the block is constructible: + let _ = System::build(|s| { + let _ = s.block(Distinct::::new(s.input(), s.output())); + }); + } + + #[test] + #[ignore = "requires stdin"] + fn run_distinct_stdout() { + use super::*; + use crate::SysBlocks; + use protoflow_core::SystemBuilding; + use tracing::error; + + if let Err(e) = System::run(|s| { + let stdin = s.read_stdin(); + let distinct = s.block(Distinct::new(s.input(), s.output())); + s.connect(&stdin.output, &distinct.input); + + let stdout_1 = s.write_stdout(); + s.connect(&distinct.output, &stdout_1.input); + }) { + error!("{}", e) + } + } +} diff --git a/lib/protoflow-blocks/src/lib.rs b/lib/protoflow-blocks/src/lib.rs index ca1f36d7..7f3b37b5 100644 --- a/lib/protoflow-blocks/src/lib.rs +++ b/lib/protoflow-blocks/src/lib.rs @@ -61,6 +61,7 @@ pub fn build_stdio_system( "Random" => Random::::build_system(config)?, // FlowBlocks "Concat" => Concat::::build_system(config)?, + "Distinct" => Distinct::::build_system(config)?, "Merge" => Merge::::build_system(config)?, "Replicate" => Replicate::::build_system(config)?, "Sort" => Sort::::build_system(config)?, diff --git a/lib/protoflow-blocks/src/system.rs b/lib/protoflow-blocks/src/system.rs index 85a9aa96..67f158b2 100644 --- a/lib/protoflow-blocks/src/system.rs +++ b/lib/protoflow-blocks/src/system.rs @@ -6,10 +6,10 @@ use crate::{ prelude::{fmt, Arc, Box, FromStr, Rc, String, ToString}, types::{DelayType, Encoding}, AllBlocks, Buffer, Concat, ConcatStrings, Const, CoreBlocks, Count, Decode, DecodeCsv, - DecodeHex, DecodeJson, Delay, Drop, Encode, EncodeCsv, EncodeHex, EncodeJson, FlowBlocks, - HashBlocks, IoBlocks, MathBlocks, Merge, Random, ReadDir, ReadEnv, ReadFile, ReadSocket, - ReadStdin, Replicate, Sort, Split, SplitString, SysBlocks, TextBlocks, WriteFile, WriteSocket, - WriteStderr, WriteStdout, + DecodeHex, DecodeJson, Delay, Distinct, Drop, Encode, EncodeCsv, EncodeHex, EncodeJson, + FlowBlocks, HashBlocks, IoBlocks, MathBlocks, Merge, Random, ReadDir, ReadEnv, ReadFile, + ReadSocket, ReadStdin, Replicate, Sort, Split, SplitString, SysBlocks, TextBlocks, WriteFile, + WriteSocket, WriteStderr, WriteStdout, }; use protoflow_core::{ Block, BlockID, BlockResult, BoxedBlockType, InputPort, Message, OutputPort, PortID, @@ -169,6 +169,10 @@ impl FlowBlocks for System { self.0.block(Concat::::with_system(self)) } + fn distinct + PartialEq + 'static>(&mut self) -> Distinct { + self.0.block(Distinct::::with_system(self)) + } + fn merge + 'static>(&mut self) -> Merge { self.0.block(Merge::::with_system(self)) } From 446391c6017f65a0082e7fc47aa6eaca9d15bbf5 Mon Sep 17 00:00:00 2001 From: evren Date: Tue, 10 Dec 2024 14:22:18 +0300 Subject: [PATCH 11/24] batch block --- README.md | 24 +++ lib/protoflow-blocks/doc/flow/batch.mmd | 11 ++ lib/protoflow-blocks/doc/flow/batch.seq.mmd | 26 +++ lib/protoflow-blocks/src/block_config.rs | 2 +- lib/protoflow-blocks/src/block_tag.rs | 4 + lib/protoflow-blocks/src/blocks/flow.rs | 14 ++ lib/protoflow-blocks/src/blocks/flow/batch.rs | 157 ++++++++++++++++++ lib/protoflow-blocks/src/lib.rs | 1 + lib/protoflow-blocks/src/system.rs | 6 +- 9 files changed, 243 insertions(+), 2 deletions(-) create mode 100644 lib/protoflow-blocks/doc/flow/batch.mmd create mode 100644 lib/protoflow-blocks/doc/flow/batch.seq.mmd create mode 100644 lib/protoflow-blocks/src/blocks/flow/batch.rs diff --git a/README.md b/README.md index 715196c4..434d7217 100644 --- a/README.md +++ b/README.md @@ -112,6 +112,7 @@ The built-in blocks provided by Protoflow are listed below: | Block | Description | |:------------------|:-------------------------------------------------------------------------------------------------------------------------------| +| [`Batch`] | Batches input strem into chunks of a specified size. | | [`Buffer`] | Stores all messages it receives. | | [`ConcatStrings`] | Concatenates the received string messages, with an optional delimiter string inserted between each message. | | [`Concat`] | Concatenates multiple input message streams into a single output stream. | @@ -145,6 +146,28 @@ The built-in blocks provided by Protoflow are listed below: | [`WriteStderr`] | Writes bytes to standard error (aka stderr). | | [`WriteStdout`] | Writes bytes to standard output (aka stdout). | +#### [`Batch`] + +A block that simply stores all messages it receives. + +```mermaid +block-beta + columns 7 + Source space:2 Batch space:2 Sink + Source-- "input" -->Batch + Batch-- "output" -->Sink + + classDef block height:48px,padding:8px; + classDef hidden visibility:none; + class Batch block + class Source hidden + class Sink hidden +``` + +```bash +protoflow execute Batch +``` + #### [`Buffer`] A block that simply stores all messages it receives. @@ -874,6 +897,7 @@ To add a new block type implementation, make sure to examine and amend: [`echo_lines`]: lib/protoflow/examples/echo_lines [`examples`]: lib/protoflow/examples +[`Batch`]: https://docs.rs/protoflow-blocks/latest/protoflow_blocks/struct.Batch.html [`Buffer`]: https://docs.rs/protoflow-blocks/latest/protoflow_blocks/struct.Buffer.html [`Concat`]: https://docs.rs/protoflow-blocks/latest/protoflow_blocks/struct.Concat.html [`ConcatStrings`]: https://docs.rs/protoflow-blocks/latest/protoflow_blocks/struct.ConcatStrings.html diff --git a/lib/protoflow-blocks/doc/flow/batch.mmd b/lib/protoflow-blocks/doc/flow/batch.mmd new file mode 100644 index 00000000..6e702128 --- /dev/null +++ b/lib/protoflow-blocks/doc/flow/batch.mmd @@ -0,0 +1,11 @@ +block-beta + columns 7 + Source space:2 Batch space:2 Sink + Source-- "input" -->Batch + Batch-- "output" -->Sink + + classDef block height:48px,padding:8px; + classDef hidden visibility:none; + class Batch block + class Source hidden + class Sink hidden diff --git a/lib/protoflow-blocks/doc/flow/batch.seq.mmd b/lib/protoflow-blocks/doc/flow/batch.seq.mmd new file mode 100644 index 00000000..0a28ee2e --- /dev/null +++ b/lib/protoflow-blocks/doc/flow/batch.seq.mmd @@ -0,0 +1,26 @@ +sequenceDiagram + autonumber + participant BlockA as Another block + participant Batch.input as Batch.input port + participant Batch as Batch block + participant Batch.output as Batch.output port + participant BlockB as Another block + + BlockA-->>Batch: Connect + Batch-->>BlockB: Connect + + loop Batch process + loop loop Until batch size is reached + BlockA->>Batch: Message + Batch->>Batch: Store Batch messages + end + loop Send buffered messages + Batch->>BlockB: Message + end + end + + + BlockA-->>Batch: Disconnect + Batch-->>Batch.input: Close + Batch-->>Batch.output: Close + Batch-->>BlockB: Disconnect diff --git a/lib/protoflow-blocks/src/block_config.rs b/lib/protoflow-blocks/src/block_config.rs index fffb27d4..11955a67 100644 --- a/lib/protoflow-blocks/src/block_config.rs +++ b/lib/protoflow-blocks/src/block_config.rs @@ -57,7 +57,7 @@ impl<'de> serde::Deserialize<'de> for BlockConfig { .unwrap() } - "Concat" | "Distinct" | "Merge" | "Replicate" | "Sort" | "Split" => { + "Batch" | "Concat" | "Distinct" | "Merge" | "Replicate" | "Sort" | "Split" => { FlowBlockConfig::deserialize(value.clone()) .map(BlockConfig::Flow) .unwrap() diff --git a/lib/protoflow-blocks/src/block_tag.rs b/lib/protoflow-blocks/src/block_tag.rs index 0b627649..6bc25320 100644 --- a/lib/protoflow-blocks/src/block_tag.rs +++ b/lib/protoflow-blocks/src/block_tag.rs @@ -18,6 +18,7 @@ pub enum BlockTag { Drop, Random, // FlowBlocks + Batch, Concat, Distinct, Merge, @@ -78,6 +79,7 @@ impl BlockTag { pub fn as_str(&self) -> &'static str { use BlockTag::*; match self { + Batch => "Batch", Buffer => "Buffer", Concat => "Concat", Const => "Const", @@ -135,6 +137,7 @@ impl FromStr for BlockTag { fn from_str(input: &str) -> Result { use BlockTag::*; Ok(match input { + "Batch" => Batch, "Buffer" => Buffer, "Concat" => Concat, "Const" => Const, @@ -203,6 +206,7 @@ impl BlockInstantiation for BlockTag { fn instantiate(&self, system: &mut System) -> Box { use BlockTag::*; match self { + Batch => Box::new(super::Batch::::with_system(system, None)), Buffer => Box::new(super::Buffer::::with_system(system)), Concat => Box::new(super::Concat::::with_system(system)), Const => Box::new(super::Const::::with_system(system, String::new())), diff --git a/lib/protoflow-blocks/src/blocks/flow.rs b/lib/protoflow-blocks/src/blocks/flow.rs index 3a03056d..8dc3e368 100644 --- a/lib/protoflow-blocks/src/blocks/flow.rs +++ b/lib/protoflow-blocks/src/blocks/flow.rs @@ -11,6 +11,7 @@ pub mod flow { use protoflow_core::{Block, ComparableAny, Message}; pub trait FlowBlocks { + fn batch + 'static>(&mut self, batch_size: usize) -> Batch; fn concat + 'static>(&mut self) -> Concat; fn distinct + PartialEq + 'static>(&mut self) -> Distinct; fn merge + 'static>(&mut self) -> Merge; @@ -22,6 +23,7 @@ pub mod flow { #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] #[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] pub enum FlowBlockTag { + Batch, Concat, Distinct, Merge, @@ -33,6 +35,10 @@ pub mod flow { #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] #[derive(Clone, Debug)] pub enum FlowBlockConfig { + Batch { + input: InputPortName, + output: OutputPortName, + }, Concat { input_1: InputPortName, input_2: InputPortName, @@ -67,6 +73,7 @@ pub mod flow { fn name(&self) -> Cow { use FlowBlockConfig::*; Cow::Borrowed(match self { + Batch { .. } => "Batch", Concat { .. } => "Concat", Distinct { .. } => "Distinct", Merge { .. } => "Merge", @@ -81,6 +88,9 @@ pub mod flow { fn output_connections(&self) -> Vec<(&'static str, Option)> { use FlowBlockConfig::*; match self { + Batch { output, .. } => { + vec![("output", Some(output.clone()))] + } Concat { output, .. } => { vec![("output", Some(output.clone()))] } @@ -118,6 +128,7 @@ pub mod flow { use super::SystemBuilding; use FlowBlockConfig::*; match self { + Batch { .. } => Box::new(super::Batch::new(system.input_any(), system.output())), Concat { .. } => Box::new(super::Concat::new( system.input_any(), system.input_any(), @@ -149,6 +160,9 @@ pub mod flow { } } + mod batch; + pub use batch::*; + mod concat; pub use concat::*; diff --git a/lib/protoflow-blocks/src/blocks/flow/batch.rs b/lib/protoflow-blocks/src/blocks/flow/batch.rs new file mode 100644 index 00000000..457cd4f2 --- /dev/null +++ b/lib/protoflow-blocks/src/blocks/flow/batch.rs @@ -0,0 +1,157 @@ +// This is free and unencumbered software released into the public domain. + +use crate::{StdioConfig, StdioError, StdioSystem, System}; +use protoflow_core::{ + prelude::{vec, Vec}, + types::Any, + Block, BlockResult, BlockRuntime, InputPort, Message, OutputPort, +}; +use protoflow_derive::Block; +use simple_mermaid::mermaid; + +/// Batches input strem into chunks of a specified size. +/// +/// # Block Diagram +#[doc = mermaid!("../../../doc/flow/batch.mmd")] +/// +/// # Sequence Diagram +#[doc = mermaid!("../../../doc/flow/batch.seq.mmd" framed)] +/// +/// # Examples +/// +/// ## Using the block in a system +/// +/// ```rust +/// # use protoflow_blocks::*; +/// # fn main() { +/// System::build(|s| { +/// let stdin = s.read_stdin(); +/// let batch = s.batch().batch(2); +/// s.connect(&stdin.output, &batch.input); +/// }); +/// # } +/// ``` +/// +/// ## Running the block via the CLI +/// +/// ```console +/// $ protoflow execute Batch +/// ``` +/// +#[derive(Block, Clone)] +pub struct Batch { + /// The input message stream. + #[input] + pub input: InputPort, + + /// The output message stream. + #[output] + pub output: OutputPort, + + /// Batch size + #[parameter] + pub batch_size: usize, + + /// The internal state storing the messages received. + #[state] + messages: Vec, +} + +impl Batch { + pub fn new(input: InputPort, output: OutputPort) -> Self { + Self::with_params(input, output, None) + } + pub fn with_params( + input: InputPort, + output: OutputPort, + batch_size: Option, + ) -> Self { + Self { + input, + output, + batch_size: batch_size.unwrap_or(1), + messages: Vec::new(), + } + } + pub fn messages(&self) -> &Vec { + &self.messages + } +} + +impl Batch { + pub fn with_system(system: &System, batch_size: Option) -> Self { + use crate::SystemBuilding; + Self::with_params(system.input(), system.output(), batch_size) + } +} + +impl Block for Batch { + fn execute(&mut self, _runtime: &dyn BlockRuntime) -> BlockResult { + while let Some(message) = self.input.recv()? { + self.messages.push(message); + + if self.batch_size == self.messages().len() { + for message in self.messages.drain(..) { + self.output.send(&message)? + } + } + } + + //send remaining messages + for message in self.messages.drain(..) { + self.output.send(&message)? + } + + Ok(()) + } +} + +#[cfg(feature = "std")] +impl StdioSystem for Batch { + fn build_system(config: StdioConfig) -> Result { + use crate::SystemBuilding; + + config.allow_only(vec!["batch-size"])?; + + Ok(System::build(|s| { + let batch_size = config.get::("batch-size").unwrap_or(1); + let stdin = config.read_stdin(s); + let batch = Batch::with_system(&s, Some(batch_size)); + s.connect(&stdin.output, &batch.input); + })) + } +} + +#[cfg(test)] +mod tests { + use super::Batch; + use crate::{FlowBlocks, System, SystemBuilding}; + + #[test] + fn instantiate_block() { + // Check that the block is constructible: + let _ = System::build(|s| { + let _ = s.block(Batch::::new(s.input(), s.output())); + }); + } + + #[test] + #[ignore = "requires stdin"] + fn run_batch_stdout() { + use super::*; + use crate::SysBlocks; + use protoflow_core::SystemBuilding; + use tracing::error; + + if let Err(e) = System::run(|s| { + let stdin = s.read_stdin(); + let batch = s.batch(2); + s.connect(&stdin.output, &batch.input); + + let stdout_1 = s.write_stdout(); + s.connect(&batch.output, &stdout_1.input); + }) { + error!("{}", e) + } + } +} diff --git a/lib/protoflow-blocks/src/lib.rs b/lib/protoflow-blocks/src/lib.rs index 7f3b37b5..612cecb3 100644 --- a/lib/protoflow-blocks/src/lib.rs +++ b/lib/protoflow-blocks/src/lib.rs @@ -53,6 +53,7 @@ pub fn build_stdio_system( use prelude::String; Ok(match system_name.as_ref() { // CoreBlocks + "Batch" => Batch::::build_system(config)?, "Buffer" => Buffer::::build_system(config)?, "Const" => Const::::build_system(config)?, "Count" => Count::::build_system(config)?, diff --git a/lib/protoflow-blocks/src/system.rs b/lib/protoflow-blocks/src/system.rs index 67f158b2..6ac67244 100644 --- a/lib/protoflow-blocks/src/system.rs +++ b/lib/protoflow-blocks/src/system.rs @@ -5,7 +5,7 @@ use crate::{ prelude::{fmt, Arc, Box, FromStr, Rc, String, ToString}, types::{DelayType, Encoding}, - AllBlocks, Buffer, Concat, ConcatStrings, Const, CoreBlocks, Count, Decode, DecodeCsv, + AllBlocks, Batch, Buffer, Concat, ConcatStrings, Const, CoreBlocks, Count, Decode, DecodeCsv, DecodeHex, DecodeJson, Delay, Distinct, Drop, Encode, EncodeCsv, EncodeHex, EncodeJson, FlowBlocks, HashBlocks, IoBlocks, MathBlocks, Merge, Random, ReadDir, ReadEnv, ReadFile, ReadSocket, ReadStdin, Replicate, Sort, Split, SplitString, SysBlocks, TextBlocks, WriteFile, @@ -165,6 +165,10 @@ impl CoreBlocks for System { } impl FlowBlocks for System { + fn batch + 'static>(&mut self, batch_size: usize) -> Batch { + self.0 + .block(Batch::::with_system(self, Some(batch_size))) + } fn concat + 'static>(&mut self) -> Concat { self.0.block(Concat::::with_system(self)) } From 390cd4b4cae3305988db5f5dd7ba309bd56c3ce1 Mon Sep 17 00:00:00 2001 From: evren Date: Fri, 13 Dec 2024 11:10:14 +0300 Subject: [PATCH 12/24] remove duplicate hash feature configurations --- lib/protoflow-blocks/src/block_config.rs | 6 ------ 1 file changed, 6 deletions(-) diff --git a/lib/protoflow-blocks/src/block_config.rs b/lib/protoflow-blocks/src/block_config.rs index 8e32c0f0..a6c86827 100644 --- a/lib/protoflow-blocks/src/block_config.rs +++ b/lib/protoflow-blocks/src/block_config.rs @@ -30,12 +30,6 @@ pub enum BlockConfig { feature = "hash-sha1", feature = "hash-sha2" ))] - #[cfg(any( - feature = "hash-blake3", - feature = "hash-md5", - feature = "hash-sha1", - feature = "hash-sha2" - ))] Hash(HashBlockConfig), Io(IoBlockConfig), Math(MathBlockConfig), From fbba02f15023354ba3e172c1ca6c157823de87ee Mon Sep 17 00:00:00 2001 From: evren Date: Fri, 13 Dec 2024 11:15:32 +0300 Subject: [PATCH 13/24] fix hash feature bug in another branch/PR --- lib/protoflow-blocks/src/block_config.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/lib/protoflow-blocks/src/block_config.rs b/lib/protoflow-blocks/src/block_config.rs index c30598ba..20897a3f 100644 --- a/lib/protoflow-blocks/src/block_config.rs +++ b/lib/protoflow-blocks/src/block_config.rs @@ -30,6 +30,12 @@ pub enum BlockConfig { feature = "hash-sha1", feature = "hash-sha2" ))] + #[cfg(any( + feature = "hash-blake3", + feature = "hash-md5", + feature = "hash-sha1", + feature = "hash-sha2" + ))] Hash(HashBlockConfig), Io(IoBlockConfig), Math(MathBlockConfig), From d046bfbc55a72d0d88628b7e9db69050beac53b9 Mon Sep 17 00:00:00 2001 From: evren Date: Fri, 13 Dec 2024 11:21:53 +0300 Subject: [PATCH 14/24] fix Concat it readme --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 939a26c2..bb77d25e 100644 --- a/README.md +++ b/README.md @@ -114,7 +114,7 @@ The built-in blocks provided by Protoflow are listed below: |:------------------|:-------------------------------------------------------------------------------------------------------------------------------| | [`Buffer`] | Stores all messages it receives. | | [`ConcatStrings`] | Concatenates the received string messages, with an optional delimiter string inserted between each message. | -| [`Concat`] | Merges multiple input message streams into a single output stream. | +| [`Concat`] | Concatenates multiple input message streams into a single output stream. | | [`Const`] | Sends a constant value. | | [`Count`] | Counts the number of messages it receives, while optionally passing them through. | | [`Decode`] | Decodes messages from a byte stream. | From 5b55a23b21c437ad2a9318837d7a91f130c5f58f Mon Sep 17 00:00:00 2001 From: evren Date: Fri, 13 Dec 2024 11:22:11 +0300 Subject: [PATCH 15/24] fix Concat it readme --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index bb77d25e..c79ab77a 100644 --- a/README.md +++ b/README.md @@ -113,8 +113,8 @@ The built-in blocks provided by Protoflow are listed below: | Block | Description | |:------------------|:-------------------------------------------------------------------------------------------------------------------------------| | [`Buffer`] | Stores all messages it receives. | -| [`ConcatStrings`] | Concatenates the received string messages, with an optional delimiter string inserted between each message. | | [`Concat`] | Concatenates multiple input message streams into a single output stream. | +| [`ConcatStrings`] | Concatenates the received string messages, with an optional delimiter string inserted between each message. | | [`Const`] | Sends a constant value. | | [`Count`] | Counts the number of messages it receives, while optionally passing them through. | | [`Decode`] | Decodes messages from a byte stream. | From ec102fd9bf8263f2af57b33df3bf91d9f6bfd0ab Mon Sep 17 00:00:00 2001 From: evren Date: Fri, 13 Dec 2024 11:28:02 +0300 Subject: [PATCH 16/24] compare values instead of types --- lib/protoflow-core/src/comparable_any.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/protoflow-core/src/comparable_any.rs b/lib/protoflow-core/src/comparable_any.rs index 52c8f55b..9e9e4847 100644 --- a/lib/protoflow-core/src/comparable_any.rs +++ b/lib/protoflow-core/src/comparable_any.rs @@ -12,13 +12,13 @@ pub struct ComparableAny(pub Any); impl PartialOrd for ComparableAny { fn partial_cmp(&self, other: &Self) -> Option { - Some(self.0.type_url.cmp(&other.0.type_url)) + Some(self.0.value.cmp(&other.0.value)) } } impl PartialEq for ComparableAny { fn eq(&self, other: &Self) -> bool { - self.0.type_url == other.0.type_url + self.0.value == other.0.value } } From cb280b710deb860de4a7d8a778a135ce32a23c8c Mon Sep 17 00:00:00 2001 From: evren Date: Fri, 13 Dec 2024 14:01:07 +0300 Subject: [PATCH 17/24] serde, std, tracing feature gates --- lib/protoflow-blocks/src/block_tag.rs | 16 ++++++------- .../src/blocks/io/decode_hex.rs | 2 ++ lib/protoflow-blocks/src/blocks/sys.rs | 23 +++++++++++++++++-- .../src/blocks/sys/read_socket.rs | 11 ++++++++- .../src/blocks/sys/write_socket.rs | 4 ++++ lib/protoflow-blocks/src/lib.rs | 2 ++ lib/protoflow-blocks/src/system.rs | 8 +++++-- 7 files changed, 53 insertions(+), 13 deletions(-) diff --git a/lib/protoflow-blocks/src/block_tag.rs b/lib/protoflow-blocks/src/block_tag.rs index 8b97ebe7..546a3f46 100644 --- a/lib/protoflow-blocks/src/block_tag.rs +++ b/lib/protoflow-blocks/src/block_tag.rs @@ -41,13 +41,13 @@ pub enum BlockTag { ReadEnv, #[cfg(feature = "std")] ReadFile, - #[cfg(feature = "std")] + #[cfg(all(feature = "std", feature = "serde"))] ReadSocket, #[cfg(feature = "std")] ReadStdin, #[cfg(feature = "std")] WriteFile, - #[cfg(feature = "std")] + #[cfg(all(feature = "std", feature = "serde"))] WriteSocket, #[cfg(feature = "std")] WriteStderr, @@ -97,13 +97,13 @@ impl BlockTag { ReadEnv => "ReadEnv", #[cfg(feature = "std")] ReadFile => "ReadFile", - #[cfg(feature = "std")] + #[cfg(all(feature = "std", feature = "serde"))] ReadSocket => "ReadSocket", #[cfg(feature = "std")] ReadStdin => "ReadStdin", #[cfg(feature = "std")] WriteFile => "WriteFile", - #[cfg(feature = "std")] + #[cfg(all(feature = "std", feature = "serde"))] WriteSocket => "WriteSocket", #[cfg(feature = "std")] WriteStderr => "WriteStderr", @@ -148,13 +148,13 @@ impl FromStr for BlockTag { "ReadEnv" => ReadEnv, #[cfg(feature = "std")] "ReadFile" => ReadFile, - #[cfg(feature = "std")] + #[cfg(all(feature = "std", feature = "serde"))] "ReadSocket" => ReadSocket, #[cfg(feature = "std")] "ReadStdin" => ReadStdin, #[cfg(feature = "std")] "WriteFile" => WriteFile, - #[cfg(feature = "std")] + #[cfg(all(feature = "std", feature = "serde"))] "WriteSocket" => WriteSocket, #[cfg(feature = "std")] "WriteStderr" => WriteStderr, @@ -210,13 +210,13 @@ impl BlockInstantiation for BlockTag { ReadEnv => Box::new(super::ReadEnv::::with_system(system)), #[cfg(feature = "std")] ReadFile => Box::new(super::ReadFile::with_system(system)), - #[cfg(feature = "std")] + #[cfg(all(feature = "std", feature = "serde"))] ReadSocket => Box::new(super::ReadSocket::with_system(system, None)), #[cfg(feature = "std")] ReadStdin => Box::new(super::ReadStdin::with_system(system, None)), #[cfg(feature = "std")] WriteFile => Box::new(super::WriteFile::with_system(system, None)), - #[cfg(feature = "std")] + #[cfg(all(feature = "std", feature = "serde"))] WriteSocket => Box::new(super::WriteSocket::with_system(system, None)), #[cfg(feature = "std")] WriteStderr => Box::new(super::WriteStderr::with_system(system)), diff --git a/lib/protoflow-blocks/src/blocks/io/decode_hex.rs b/lib/protoflow-blocks/src/blocks/io/decode_hex.rs index b61116df..e17b56c0 100644 --- a/lib/protoflow-blocks/src/blocks/io/decode_hex.rs +++ b/lib/protoflow-blocks/src/blocks/io/decode_hex.rs @@ -7,6 +7,7 @@ use crate::{ use protoflow_core::{Block, BlockError, BlockResult, BlockRuntime, InputPort, OutputPort}; use protoflow_derive::Block; use simple_mermaid::mermaid; +#[cfg(feature = "tracing")] use tracing; /// A block that decodes a hexadecimal byte stream to byte. @@ -95,6 +96,7 @@ fn hex_value(byte: u8) -> Result { b'A'..=b'F' => Ok(byte - b'A' + 10), _ => { let err = format!("Invalid hex character: '{}' (0x{:02X})", byte as char, byte); + #[cfg(feature = "tracing")] tracing::error!(target: "DecodeHex:hex_value", err); Err(BlockError::Other(err)) } diff --git a/lib/protoflow-blocks/src/blocks/sys.rs b/lib/protoflow-blocks/src/blocks/sys.rs index 336eb2ad..71e55bec 100644 --- a/lib/protoflow-blocks/src/blocks/sys.rs +++ b/lib/protoflow-blocks/src/blocks/sys.rs @@ -19,9 +19,11 @@ pub mod sys { fn read_dir(&mut self) -> ReadDir; fn read_env(&mut self) -> ReadEnv; fn read_file(&mut self) -> ReadFile; + #[cfg(feature = "serde")] fn read_socket(&mut self) -> ReadSocket; fn read_stdin(&mut self) -> ReadStdin; fn write_file(&mut self) -> WriteFile; + #[cfg(feature = "serde")] fn write_socket(&mut self) -> WriteSocket; fn write_stderr(&mut self) -> WriteStderr; fn write_stdout(&mut self) -> WriteStdout; @@ -33,9 +35,11 @@ pub mod sys { ReadDir, ReadEnv, ReadFile, + #[cfg(feature = "serde")] ReadSocket, ReadStdin, WriteFile, + #[cfg(feature = "serde")] WriteSocket, WriteStderr, WriteStdout, @@ -59,6 +63,7 @@ pub mod sys { output: OutputPortName, }, + #[cfg(feature = "serde")] ReadSocket { output: OutputPortName, config: ReadSocketConfig, @@ -75,6 +80,7 @@ pub mod sys { flags: Option, }, + #[cfg(feature = "serde")] WriteSocket { input: InputPortName, config: WriteSocketConfig, @@ -96,9 +102,11 @@ pub mod sys { ReadDir { .. } => "ReadDir", ReadEnv { .. } => "ReadEnv", ReadFile { .. } => "ReadFile", + #[cfg(feature = "serde")] ReadSocket { .. } => "ReadSocket", ReadStdin { .. } => "ReadStdin", WriteFile { .. } => "WriteFile", + #[cfg(feature = "serde")] WriteSocket { .. } => "WriteSocket", WriteStderr { .. } => "WriteStderr", WriteStdout { .. } => "WriteStdout", @@ -113,13 +121,18 @@ pub mod sys { ReadDir { output, .. } | ReadEnv { output, .. } | ReadFile { output, .. } - | ReadSocket { output, .. } | ReadStdin { output, .. } => { vec![("output", Some(output.clone()))] } - WriteFile { .. } | WriteSocket { .. } | WriteStderr { .. } | WriteStdout { .. } => { + WriteFile { .. } | WriteStderr { .. } | WriteStdout { .. } => { vec![] } + #[cfg(feature = "serde")] + ReadSocket { output, .. } => { + vec![("output", Some(output.clone()))] + } + #[cfg(feature = "serde")] + WriteSocket { .. } => vec![], } } } @@ -131,6 +144,7 @@ pub mod sys { ReadDir { .. } => Box::new(super::ReadDir::with_system(system)), ReadEnv { .. } => Box::new(super::ReadEnv::::with_system(system)), ReadFile { .. } => Box::new(super::ReadFile::with_system(system)), + #[cfg(feature = "serde")] ReadSocket { config, .. } => { Box::new(super::ReadSocket::with_system(system, Some(config.clone()))) } @@ -138,6 +152,7 @@ pub mod sys { Box::new(super::ReadStdin::with_system(system, *buffer_size)) } WriteFile { flags, .. } => Box::new(super::WriteFile::with_system(system, *flags)), + #[cfg(feature = "serde")] WriteSocket { config, .. } => Box::new(super::WriteSocket::with_system( system, Some(config.clone()), @@ -157,7 +172,9 @@ pub mod sys { mod read_file; pub use read_file::*; + #[cfg(feature = "serde")] mod read_socket; + #[cfg(feature = "serde")] pub use read_socket::*; mod read_stdin; @@ -166,7 +183,9 @@ pub mod sys { mod write_file; pub use write_file::*; + #[cfg(feature = "serde")] mod write_socket; + #[cfg(feature = "serde")] pub use write_socket::*; mod write_stderr; diff --git a/lib/protoflow-blocks/src/blocks/sys/read_socket.rs b/lib/protoflow-blocks/src/blocks/sys/read_socket.rs index 1f51b265..c7bac354 100644 --- a/lib/protoflow-blocks/src/blocks/sys/read_socket.rs +++ b/lib/protoflow-blocks/src/blocks/sys/read_socket.rs @@ -16,6 +16,7 @@ use std::{ net::{TcpListener, TcpStream}, sync::{Arc, Mutex, PoisonError}, }; +#[cfg(feature = "tracing")] use tracing::{error, info}; /// A block that reads a proto object from a TCP port. @@ -100,6 +101,7 @@ impl Block for ReadSocket { fn prepare(&mut self, _runtime: &dyn BlockRuntime) -> BlockResult { let listener = TcpListener::bind(&self.config.connection)?; *self.listener.lock().map_err(lock_error)? = Some(listener); + #[cfg(feature = "tracing")] info!("Server listening on {}", &self.config.connection); Ok(()) } @@ -114,16 +116,18 @@ impl Block for ReadSocket { .ok_or(BlockError::Other("Invalid TCP listener".into()))?; let (stream, addr) = listener.accept().map_err(|e| { + #[cfg(feature = "tracing")] error!("Failed to accept client connection: {}", e); BlockError::Other("Failed to accept client connection".into()) })?; - + #[cfg(feature = "tracing")] info!("Accepted connection from {}", addr); *stream_guard = Some(stream); } if let Some(stream) = stream_guard.as_mut() { handle_client::<_>(stream, self.config.buffer_size, |message| { + #[cfg(feature = "tracing")] info!("Processing received message"); if self.output.is_connected() { self.output.send(message)?; @@ -131,6 +135,7 @@ impl Block for ReadSocket { Ok(()) }) .map_err(|e| { + #[cfg(feature = "tracing")] error!("Error handling client: {}", e); BlockError::Other("Error handling client".into()) })?; @@ -158,14 +163,17 @@ where let bytes_read = stream.read(&mut buffer)?; if bytes_read == 0 { + #[cfg(feature = "tracing")] info!("Client disconnected"); break; } let message = Bytes::copy_from_slice(&buffer[..bytes_read]); + #[cfg(feature = "tracing")] info!("Received message: {:?}", message); if let Err(e) = process_fn(&message) { + #[cfg(feature = "tracing")] error!("Failed to process message: {:?}", e); return Err(BlockError::Other("Failed to process message".into())); } @@ -215,6 +223,7 @@ pub mod read_socket_tests { )); s.connect(&read_socket.output, &std_out.input); }) { + #[cfg(feature = "tracing")] error!("{}", e) } } diff --git a/lib/protoflow-blocks/src/blocks/sys/write_socket.rs b/lib/protoflow-blocks/src/blocks/sys/write_socket.rs index 3bc998ad..a21f6d3e 100644 --- a/lib/protoflow-blocks/src/blocks/sys/write_socket.rs +++ b/lib/protoflow-blocks/src/blocks/sys/write_socket.rs @@ -14,6 +14,7 @@ use std::{ net::TcpStream, sync::{Arc, Mutex, PoisonError}, }; +#[cfg(feature = "tracing")] use tracing::error; /// A block that writes a proto object to a TCP socket. @@ -98,6 +99,7 @@ impl Block for WriteSocket { if stream_guard.is_none() { *stream_guard = Some(TcpStream::connect(&self.config.connection).map_err(|e| { + #[cfg(feature = "tracing")] error!("Failed to connect to {}: {}", &self.config.connection, e); BlockError::Other(format!( "Failed to connect to {}: {}", @@ -107,6 +109,7 @@ impl Block for WriteSocket { } let stream = stream_guard.as_mut().ok_or_else(|| { + #[cfg(feature = "tracing")] error!("Stream is not connected"); BlockError::Other("Stream is not connected".into()) })?; @@ -155,6 +158,7 @@ pub mod write_socket_tests { }); s.connect(&stdin.output, &write_socket.input); }) { + #[cfg(feature = "tracing")] error!("{}", e) } } diff --git a/lib/protoflow-blocks/src/lib.rs b/lib/protoflow-blocks/src/lib.rs index 30050a0b..41d1decd 100644 --- a/lib/protoflow-blocks/src/lib.rs +++ b/lib/protoflow-blocks/src/lib.rs @@ -80,9 +80,11 @@ pub fn build_stdio_system( "ReadDir" => ReadDir::build_system(config)?, "ReadEnv" => ReadEnv::::build_system(config)?, "ReadFile" => ReadFile::build_system(config)?, + #[cfg(feature = "serde")] "ReadSocket" => ReadSocket::build_system(config)?, "ReadStdin" => ReadStdin::build_system(config)?, "WriteFile" => WriteFile::build_system(config)?, + #[cfg(feature = "serde")] "WriteSocket" => WriteSocket::build_system(config)?, "WriteStderr" => WriteStderr::build_system(config)?, "WriteStdout" => WriteStdout::build_system(config)?, diff --git a/lib/protoflow-blocks/src/system.rs b/lib/protoflow-blocks/src/system.rs index 695ccfba..f66363b5 100644 --- a/lib/protoflow-blocks/src/system.rs +++ b/lib/protoflow-blocks/src/system.rs @@ -7,9 +7,11 @@ use crate::{ types::{DelayType, Encoding}, AllBlocks, Buffer, ConcatStrings, Const, CoreBlocks, Count, Decode, DecodeCsv, DecodeHex, DecodeJson, Delay, Drop, Encode, EncodeCsv, EncodeHex, EncodeJson, FlowBlocks, HashBlocks, - IoBlocks, MathBlocks, Random, ReadDir, ReadEnv, ReadFile, ReadSocket, ReadStdin, SplitString, - SysBlocks, TextBlocks, WriteFile, WriteSocket, WriteStderr, WriteStdout, + IoBlocks, MathBlocks, Random, ReadDir, ReadEnv, ReadFile, ReadStdin, SplitString, SysBlocks, + TextBlocks, WriteFile, WriteStderr, WriteStdout, }; +#[cfg(all(feature = "std", feature = "serde"))] +use crate::{ReadSocket, WriteSocket}; use protoflow_core::{ Block, BlockID, BlockResult, BoxedBlockType, InputPort, Message, OutputPort, PortID, PortResult, Process, SystemBuilding, SystemExecution, @@ -258,6 +260,7 @@ impl SysBlocks for System { self.0.block(ReadFile::with_system(self)) } + #[cfg(feature = "serde")] fn read_socket(&mut self) -> ReadSocket { self.0.block(ReadSocket::with_system(self, None)) } @@ -270,6 +273,7 @@ impl SysBlocks for System { self.0.block(WriteFile::with_system(self, None)) } + #[cfg(feature = "serde")] fn write_socket(&mut self) -> WriteSocket { self.0.block(WriteSocket::with_system(self, None)) } From a82f4368c3863cdc63ef4895d35bb1c495e83c0f Mon Sep 17 00:00:00 2001 From: evren Date: Fri, 13 Dec 2024 15:57:28 +0300 Subject: [PATCH 18/24] tracing feature gate --- lib/protoflow-blocks/src/blocks/flow/batch.rs | 2 ++ lib/protoflow-blocks/src/blocks/flow/concat.rs | 8 ++++++++ lib/protoflow-blocks/src/blocks/flow/distinct.rs | 2 ++ lib/protoflow-blocks/src/blocks/flow/merge.rs | 5 +++++ lib/protoflow-blocks/src/blocks/flow/sort.rs | 4 ++++ lib/protoflow-blocks/src/blocks/flow/split.rs | 3 +++ 6 files changed, 24 insertions(+) diff --git a/lib/protoflow-blocks/src/blocks/flow/batch.rs b/lib/protoflow-blocks/src/blocks/flow/batch.rs index 457cd4f2..9b5f5d59 100644 --- a/lib/protoflow-blocks/src/blocks/flow/batch.rs +++ b/lib/protoflow-blocks/src/blocks/flow/batch.rs @@ -141,6 +141,7 @@ mod tests { use super::*; use crate::SysBlocks; use protoflow_core::SystemBuilding; + #[cfg(feature = "tracing")] use tracing::error; if let Err(e) = System::run(|s| { @@ -151,6 +152,7 @@ mod tests { let stdout_1 = s.write_stdout(); s.connect(&batch.output, &stdout_1.input); }) { + #[cfg(feature = "tracing")] error!("{}", e) } } diff --git a/lib/protoflow-blocks/src/blocks/flow/concat.rs b/lib/protoflow-blocks/src/blocks/flow/concat.rs index cc3fd4f6..b5f77a41 100644 --- a/lib/protoflow-blocks/src/blocks/flow/concat.rs +++ b/lib/protoflow-blocks/src/blocks/flow/concat.rs @@ -91,6 +91,7 @@ impl Block for Concat { while let Ok(Some(message)) = input.recv() { buffer.push(message); } + #[cfg(feature = "tracing")] tracing::info!("{} processed {} messages", input_name, buffer.len()); Ok(buffer) } @@ -109,10 +110,12 @@ impl Block for Concat { // Collect and handle thread results let buffer1 = match handle1.join() { Ok(result) => result.map_err(|e| { + #[cfg(feature = "tracing")] tracing::error!("Error processing input1: {:?}", e); BlockError::Other("Failed to process input1".into()) })?, Err(_) => { + #[cfg(feature = "tracing")] tracing::error!("Thread for input1 panicked"); return Err(BlockError::Other("Thread for input1 panicked".into())); } @@ -120,16 +123,19 @@ impl Block for Concat { let buffer2 = match handle2.join() { Ok(result) => result.map_err(|e| { + #[cfg(feature = "tracing")] tracing::error!("Error processing input2: {:?}", e); BlockError::Other("Failed to process input2".into()) })?, Err(_) => { + #[cfg(feature = "tracing")] tracing::error!("Thread for input2 panicked"); return Err(BlockError::Other("Thread for input2 panicked".into())); } }; // Concatenate and send messages to the output sequentially + #[cfg(feature = "tracing")] tracing::info!( "Concatenating {} messages from input1 with {} messages from input2", buffer1.len(), @@ -138,11 +144,13 @@ impl Block for Concat { for message in buffer1.iter().chain(buffer2.iter()) { if let Err(err) = self.output.send(message) { + #[cfg(feature = "tracing")] tracing::error!("Failed to send message: {:?}", err); return Err(err.into()); } } + #[cfg(feature = "tracing")] tracing::info!("All messages successfully sent to the output."); Ok(()) } diff --git a/lib/protoflow-blocks/src/blocks/flow/distinct.rs b/lib/protoflow-blocks/src/blocks/flow/distinct.rs index d8571800..ae9050df 100644 --- a/lib/protoflow-blocks/src/blocks/flow/distinct.rs +++ b/lib/protoflow-blocks/src/blocks/flow/distinct.rs @@ -124,6 +124,7 @@ mod tests { use super::*; use crate::SysBlocks; use protoflow_core::SystemBuilding; + #[cfg(feature = "tracing")] use tracing::error; if let Err(e) = System::run(|s| { @@ -134,6 +135,7 @@ mod tests { let stdout_1 = s.write_stdout(); s.connect(&distinct.output, &stdout_1.input); }) { + #[cfg(feature = "tracing")] error!("{}", e) } } diff --git a/lib/protoflow-blocks/src/blocks/flow/merge.rs b/lib/protoflow-blocks/src/blocks/flow/merge.rs index ec0db67e..a781c0ab 100644 --- a/lib/protoflow-blocks/src/blocks/flow/merge.rs +++ b/lib/protoflow-blocks/src/blocks/flow/merge.rs @@ -88,6 +88,7 @@ impl Block for Merge { ) -> Result<(), BlockError> { while let Ok(Some(message)) = input.recv() { if let Err(err) = output.send(&message) { + #[cfg(feature = "tracing")] tracing::error!("Error sending message: {}", err); return Err(BlockError::Other(format!("Error sending message: {}", err))); } @@ -108,11 +109,13 @@ impl Block for Merge { }; if let Err(_) = input1_thread.join() { + #[cfg(feature = "tracing")] tracing::error!("Thread for input1 panicked"); return Err(BlockError::Other("Thread for input1 panicked".into())); } if let Err(_) = input2_thread.join() { + #[cfg(feature = "tracing")] tracing::error!("Thread for input2 panicked"); return Err(BlockError::Other("Thread for input2 panicked".into())); } @@ -148,6 +151,7 @@ impl StdioSystem for Merge { mod merge_tests { use crate::{FlowBlocks, SysBlocks, System}; use protoflow_core::{prelude::String, SystemBuilding}; + #[cfg(feature = "tracing")] use tracing::error; use super::Merge; @@ -178,6 +182,7 @@ mod merge_tests { let stdout_1 = s.write_stdout(); s.connect(&merge.output, &stdout_1.input); }) { + #[cfg(feature = "tracing")] error!("{}", e) } } diff --git a/lib/protoflow-blocks/src/blocks/flow/sort.rs b/lib/protoflow-blocks/src/blocks/flow/sort.rs index cdf33747..d3859e2d 100644 --- a/lib/protoflow-blocks/src/blocks/flow/sort.rs +++ b/lib/protoflow-blocks/src/blocks/flow/sort.rs @@ -8,6 +8,7 @@ use protoflow_core::{ }; use protoflow_derive::Block; use simple_mermaid::mermaid; +#[cfg(feature = "tracing")] use tracing::error; /// Sorts a single input message stream in ascending order. @@ -85,6 +86,7 @@ impl Block for Sort { if let Some(ordering) = x.partial_cmp(y) { ordering } else { + #[cfg(feature = "tracing")] error!("Incomparable values: {:?} and {:?}", x, y); Ordering::Equal } @@ -134,6 +136,7 @@ mod tests { use super::*; use crate::SysBlocks; use protoflow_core::SystemBuilding; + #[cfg(feature = "tracing")] use tracing::error; if let Err(e) = System::run(|s| { @@ -144,6 +147,7 @@ mod tests { let stdout_1 = s.write_stdout(); s.connect(&sort.output, &stdout_1.input); }) { + #[cfg(feature = "tracing")] error!("{}", e) } } diff --git a/lib/protoflow-blocks/src/blocks/flow/split.rs b/lib/protoflow-blocks/src/blocks/flow/split.rs index 9bfa4f68..d1a90334 100644 --- a/lib/protoflow-blocks/src/blocks/flow/split.rs +++ b/lib/protoflow-blocks/src/blocks/flow/split.rs @@ -114,6 +114,7 @@ impl StdioSystem for Split { mod split_tests { use crate::{CoreBlocks, FlowBlocks, SysBlocks, System}; use protoflow_core::prelude::String; + #[cfg(feature = "tracing")] use tracing::error; extern crate std; @@ -146,6 +147,7 @@ mod split_tests { s.connect(&file.output, &write_file.path); s.connect(&split.output_2, &write_file.input); }) { + #[cfg(feature = "tracing")] error!("{}", e) } } @@ -166,6 +168,7 @@ mod split_tests { let stdout_2 = s.write_stdout(); s.connect(&split.output_2, &stdout_2.input); }) { + #[cfg(feature = "tracing")] error!("{}", e) } } From 8bae9f0ff507c95a36c6b76f310f40c2e76852ff Mon Sep 17 00:00:00 2001 From: evren Date: Fri, 13 Dec 2024 16:53:12 +0300 Subject: [PATCH 19/24] logging --- lib/protoflow-blocks/src/blocks/flow/batch.rs | 5 +++++ lib/protoflow-blocks/src/blocks/flow/distinct.rs | 10 ++++++---- lib/protoflow-blocks/src/blocks/flow/replicate.rs | 2 ++ lib/protoflow-blocks/src/blocks/flow/sort.rs | 8 ++++---- 4 files changed, 17 insertions(+), 8 deletions(-) diff --git a/lib/protoflow-blocks/src/blocks/flow/batch.rs b/lib/protoflow-blocks/src/blocks/flow/batch.rs index 9b5f5d59..22f8e144 100644 --- a/lib/protoflow-blocks/src/blocks/flow/batch.rs +++ b/lib/protoflow-blocks/src/blocks/flow/batch.rs @@ -88,9 +88,13 @@ impl Batch { impl Block for Batch { fn execute(&mut self, _runtime: &dyn BlockRuntime) -> BlockResult { while let Some(message) = self.input.recv()? { + #[cfg(feature = "tracing")] + tracing::info!("Buffered one message"); self.messages.push(message); if self.batch_size == self.messages().len() { + #[cfg(feature = "tracing")] + tracing::info!("Sending messages"); for message in self.messages.drain(..) { self.output.send(&message)? } @@ -98,6 +102,7 @@ impl Block for Batch { } //send remaining messages + tracing::info!("Sending remaining messages"); for message in self.messages.drain(..) { self.output.send(&message)? } diff --git a/lib/protoflow-blocks/src/blocks/flow/distinct.rs b/lib/protoflow-blocks/src/blocks/flow/distinct.rs index ae9050df..a579672d 100644 --- a/lib/protoflow-blocks/src/blocks/flow/distinct.rs +++ b/lib/protoflow-blocks/src/blocks/flow/distinct.rs @@ -75,17 +75,19 @@ impl Distinct { impl Block for Distinct { fn execute(&mut self, _runtime: &dyn BlockRuntime) -> BlockResult { while let Some(message) = self.input.recv()? { + #[cfg(feature = "tracing")] + tracing::info!("Buffered one message"); if !self.messages.contains(&message) { self.messages.push(message); } } - for message in self.messages.iter() { - self.output.send(message)?; + #[cfg(feature = "tracing")] + tracing::info!("Sending messages"); + for message in self.messages.drain(..) { + self.output.send(&message)?; } - self.messages.clear(); - Ok(()) } } diff --git a/lib/protoflow-blocks/src/blocks/flow/replicate.rs b/lib/protoflow-blocks/src/blocks/flow/replicate.rs index 251ded76..d079f7cf 100644 --- a/lib/protoflow-blocks/src/blocks/flow/replicate.rs +++ b/lib/protoflow-blocks/src/blocks/flow/replicate.rs @@ -76,6 +76,8 @@ impl Block for Replicate { runtime.wait_for(&self.output_2)?; while let Some(message) = self.input.recv()? { + #[cfg(feature = "tracing")] + tracing::info!("Sending message"); self.output_1.send(&message)?; self.output_2.send(&message)?; } diff --git a/lib/protoflow-blocks/src/blocks/flow/sort.rs b/lib/protoflow-blocks/src/blocks/flow/sort.rs index d3859e2d..a402a2af 100644 --- a/lib/protoflow-blocks/src/blocks/flow/sort.rs +++ b/lib/protoflow-blocks/src/blocks/flow/sort.rs @@ -82,6 +82,8 @@ impl Block for Sort { self.messages.push(message); } + #[cfg(feature = "tracing")] + tracing::info!("Sorting messages"); self.messages.sort_by(|x, y| { if let Some(ordering) = x.partial_cmp(y) { ordering @@ -92,12 +94,10 @@ impl Block for Sort { } }); - for x in self.messages.iter() { - self.output.send(x)?; + for message in self.messages.drain(..) { + self.output.send(&message)?; } - self.messages.clear(); - Ok(()) } } From 056a913db77a498aa1faff604099a6fa99ee835c Mon Sep 17 00:00:00 2001 From: evren Date: Fri, 13 Dec 2024 16:54:36 +0300 Subject: [PATCH 20/24] logging --- lib/protoflow-blocks/src/blocks/flow/batch.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/protoflow-blocks/src/blocks/flow/batch.rs b/lib/protoflow-blocks/src/blocks/flow/batch.rs index 22f8e144..d6e4ea95 100644 --- a/lib/protoflow-blocks/src/blocks/flow/batch.rs +++ b/lib/protoflow-blocks/src/blocks/flow/batch.rs @@ -102,6 +102,7 @@ impl Block for Batch { } //send remaining messages + #[cfg(feature = "tracing")] tracing::info!("Sending remaining messages"); for message in self.messages.drain(..) { self.output.send(&message)? From cf7f38bf59d335e0fa3364cae4b347ac510110cb Mon Sep 17 00:00:00 2001 From: evren Date: Mon, 16 Dec 2024 09:44:54 +0300 Subject: [PATCH 21/24] remove duplicate 'concat' in readme --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index 698760b2..81a919f1 100644 --- a/README.md +++ b/README.md @@ -115,7 +115,6 @@ The built-in blocks provided by Protoflow are listed below: | [`Batch`] | Batches input strem into chunks of a specified size. | | [`Buffer`] | Stores all messages it receives. | | [`Concat`] | Concatenates multiple input message streams into a single output stream. | -| [`Concat`] | Concatenates multiple input message streams into a single output stream. | | [`ConcatStrings`] | Concatenates the received string messages, with an optional delimiter string inserted between each message. | | [`Const`] | Sends a constant value. | | [`Count`] | Counts the number of messages it receives, while optionally passing them through. | From d2f795ce508722fd53085f4668ccda7898528847 Mon Sep 17 00:00:00 2001 From: evren Date: Mon, 23 Dec 2024 10:45:43 +0300 Subject: [PATCH 22/24] use tracing macros --- lib/protoflow-blocks/src/blocks/flow/batch.rs | 15 ++++---- .../src/blocks/flow/concat.rs | 34 ++++++++----------- .../src/blocks/flow/distinct.rs | 14 +++----- lib/protoflow-blocks/src/blocks/flow/merge.rs | 26 ++++++-------- .../src/blocks/flow/replicate.rs | 5 ++- lib/protoflow-blocks/src/blocks/flow/sort.rs | 14 +++----- lib/protoflow-blocks/src/blocks/flow/split.rs | 6 +--- .../src/blocks/sys/read_socket.rs | 8 ----- .../src/blocks/sys/write_socket.rs | 3 -- 9 files changed, 42 insertions(+), 83 deletions(-) diff --git a/lib/protoflow-blocks/src/blocks/flow/batch.rs b/lib/protoflow-blocks/src/blocks/flow/batch.rs index d6e4ea95..3a6025d5 100644 --- a/lib/protoflow-blocks/src/blocks/flow/batch.rs +++ b/lib/protoflow-blocks/src/blocks/flow/batch.rs @@ -2,6 +2,7 @@ use crate::{StdioConfig, StdioError, StdioSystem, System}; use protoflow_core::{ + info, prelude::{vec, Vec}, types::Any, Block, BlockResult, BlockRuntime, InputPort, Message, OutputPort, @@ -88,13 +89,11 @@ impl Batch { impl Block for Batch { fn execute(&mut self, _runtime: &dyn BlockRuntime) -> BlockResult { while let Some(message) = self.input.recv()? { - #[cfg(feature = "tracing")] - tracing::info!("Buffered one message"); + info!("Buffered one message"); self.messages.push(message); if self.batch_size == self.messages().len() { - #[cfg(feature = "tracing")] - tracing::info!("Sending messages"); + info!("Sending messages"); for message in self.messages.drain(..) { self.output.send(&message)? } @@ -102,8 +101,7 @@ impl Block for Batch { } //send remaining messages - #[cfg(feature = "tracing")] - tracing::info!("Sending remaining messages"); + info!("Sending remaining messages"); for message in self.messages.drain(..) { self.output.send(&message)? } @@ -130,6 +128,8 @@ impl StdioSystem for Batch { #[cfg(test)] mod tests { + use protoflow_core::error; + use super::Batch; use crate::{FlowBlocks, System, SystemBuilding}; @@ -147,8 +147,6 @@ mod tests { use super::*; use crate::SysBlocks; use protoflow_core::SystemBuilding; - #[cfg(feature = "tracing")] - use tracing::error; if let Err(e) = System::run(|s| { let stdin = s.read_stdin(); @@ -158,7 +156,6 @@ mod tests { let stdout_1 = s.write_stdout(); s.connect(&batch.output, &stdout_1.input); }) { - #[cfg(feature = "tracing")] error!("{}", e) } } diff --git a/lib/protoflow-blocks/src/blocks/flow/concat.rs b/lib/protoflow-blocks/src/blocks/flow/concat.rs index b5f77a41..b9a202a6 100644 --- a/lib/protoflow-blocks/src/blocks/flow/concat.rs +++ b/lib/protoflow-blocks/src/blocks/flow/concat.rs @@ -1,12 +1,14 @@ // This is free and unencumbered software released into the public domain. extern crate std; -use crate::prelude::{Arc, Vec}; -use crate::{FlowBlocks, StdioConfig, StdioError, StdioSystem, SysBlocks, System}; +use crate::{ + prelude::{Arc, Vec}, + FlowBlocks, StdioConfig, StdioError, StdioSystem, SysBlocks, System, +}; use protoflow_core::{ - types::Any, Block, BlockResult, BlockRuntime, InputPort, Message, OutputPort, + error, info, types::Any, Block, BlockError, BlockResult, BlockRuntime, InputPort, Message, + OutputPort, PortError, }; -use protoflow_core::{BlockError, PortError}; use protoflow_derive::Block; use simple_mermaid::mermaid; @@ -91,8 +93,7 @@ impl Block for Concat { while let Ok(Some(message)) = input.recv() { buffer.push(message); } - #[cfg(feature = "tracing")] - tracing::info!("{} processed {} messages", input_name, buffer.len()); + info!("{} processed {} messages", input_name, buffer.len()); Ok(buffer) } @@ -110,33 +111,28 @@ impl Block for Concat { // Collect and handle thread results let buffer1 = match handle1.join() { Ok(result) => result.map_err(|e| { - #[cfg(feature = "tracing")] - tracing::error!("Error processing input1: {:?}", e); + error!("Error processing input1: {:?}", e); BlockError::Other("Failed to process input1".into()) })?, Err(_) => { - #[cfg(feature = "tracing")] - tracing::error!("Thread for input1 panicked"); + error!("Thread for input1 panicked"); return Err(BlockError::Other("Thread for input1 panicked".into())); } }; let buffer2 = match handle2.join() { Ok(result) => result.map_err(|e| { - #[cfg(feature = "tracing")] - tracing::error!("Error processing input2: {:?}", e); + error!("Error processing input2: {:?}", e); BlockError::Other("Failed to process input2".into()) })?, Err(_) => { - #[cfg(feature = "tracing")] - tracing::error!("Thread for input2 panicked"); + error!("Thread for input2 panicked"); return Err(BlockError::Other("Thread for input2 panicked".into())); } }; // Concatenate and send messages to the output sequentially - #[cfg(feature = "tracing")] - tracing::info!( + info!( "Concatenating {} messages from input1 with {} messages from input2", buffer1.len(), buffer2.len() @@ -144,14 +140,12 @@ impl Block for Concat { for message in buffer1.iter().chain(buffer2.iter()) { if let Err(err) = self.output.send(message) { - #[cfg(feature = "tracing")] - tracing::error!("Failed to send message: {:?}", err); + error!("Failed to send message: {:?}", err); return Err(err.into()); } } - #[cfg(feature = "tracing")] - tracing::info!("All messages successfully sent to the output."); + info!("All messages successfully sent to the output."); Ok(()) } } diff --git a/lib/protoflow-blocks/src/blocks/flow/distinct.rs b/lib/protoflow-blocks/src/blocks/flow/distinct.rs index a579672d..056334bf 100644 --- a/lib/protoflow-blocks/src/blocks/flow/distinct.rs +++ b/lib/protoflow-blocks/src/blocks/flow/distinct.rs @@ -2,7 +2,8 @@ use crate::{StdioConfig, StdioError, StdioSystem, System}; use protoflow_core::{ - prelude::Vec, types::Any, Block, BlockResult, BlockRuntime, InputPort, Message, OutputPort, + info, prelude::Vec, types::Any, Block, BlockResult, BlockRuntime, InputPort, Message, + OutputPort, }; use protoflow_derive::Block; use simple_mermaid::mermaid; @@ -75,15 +76,13 @@ impl Distinct { impl Block for Distinct { fn execute(&mut self, _runtime: &dyn BlockRuntime) -> BlockResult { while let Some(message) = self.input.recv()? { - #[cfg(feature = "tracing")] - tracing::info!("Buffered one message"); + info!("Buffered one message"); if !self.messages.contains(&message) { self.messages.push(message); } } - #[cfg(feature = "tracing")] - tracing::info!("Sending messages"); + info!("Sending messages"); for message in self.messages.drain(..) { self.output.send(&message)?; } @@ -125,9 +124,7 @@ mod tests { fn run_distinct_stdout() { use super::*; use crate::SysBlocks; - use protoflow_core::SystemBuilding; - #[cfg(feature = "tracing")] - use tracing::error; + use protoflow_core::{error, SystemBuilding}; if let Err(e) = System::run(|s| { let stdin = s.read_stdin(); @@ -137,7 +134,6 @@ mod tests { let stdout_1 = s.write_stdout(); s.connect(&distinct.output, &stdout_1.input); }) { - #[cfg(feature = "tracing")] error!("{}", e) } } diff --git a/lib/protoflow-blocks/src/blocks/flow/merge.rs b/lib/protoflow-blocks/src/blocks/flow/merge.rs index a781c0ab..903b03c4 100644 --- a/lib/protoflow-blocks/src/blocks/flow/merge.rs +++ b/lib/protoflow-blocks/src/blocks/flow/merge.rs @@ -1,11 +1,12 @@ // This is free and unencumbered software released into the public domain. extern crate std; -use crate::prelude::{format, Arc}; -use crate::{FlowBlocks, StdioConfig, StdioError, StdioSystem, SysBlocks, System}; -use protoflow_core::BlockError; +use crate::{ + prelude::{format, Arc}, + FlowBlocks, StdioConfig, StdioError, StdioSystem, SysBlocks, System, +}; use protoflow_core::{ - types::Any, Block, BlockResult, BlockRuntime, InputPort, Message, OutputPort, + error, types::Any, Block, BlockError, BlockResult, BlockRuntime, InputPort, Message, OutputPort, }; use protoflow_derive::Block; use simple_mermaid::mermaid; @@ -88,8 +89,7 @@ impl Block for Merge { ) -> Result<(), BlockError> { while let Ok(Some(message)) = input.recv() { if let Err(err) = output.send(&message) { - #[cfg(feature = "tracing")] - tracing::error!("Error sending message: {}", err); + error!("Error sending message: {}", err); return Err(BlockError::Other(format!("Error sending message: {}", err))); } } @@ -109,14 +109,12 @@ impl Block for Merge { }; if let Err(_) = input1_thread.join() { - #[cfg(feature = "tracing")] - tracing::error!("Thread for input1 panicked"); + error!("Thread for input1 panicked"); return Err(BlockError::Other("Thread for input1 panicked".into())); } if let Err(_) = input2_thread.join() { - #[cfg(feature = "tracing")] - tracing::error!("Thread for input2 panicked"); + error!("Thread for input2 panicked"); return Err(BlockError::Other("Thread for input2 panicked".into())); } @@ -149,12 +147,9 @@ impl StdioSystem for Merge { #[cfg(test)] mod merge_tests { - use crate::{FlowBlocks, SysBlocks, System}; - use protoflow_core::{prelude::String, SystemBuilding}; - #[cfg(feature = "tracing")] - use tracing::error; - use super::Merge; + use crate::{FlowBlocks, SysBlocks, System}; + use protoflow_core::{error, prelude::String, SystemBuilding}; extern crate std; @@ -182,7 +177,6 @@ mod merge_tests { let stdout_1 = s.write_stdout(); s.connect(&merge.output, &stdout_1.input); }) { - #[cfg(feature = "tracing")] error!("{}", e) } } diff --git a/lib/protoflow-blocks/src/blocks/flow/replicate.rs b/lib/protoflow-blocks/src/blocks/flow/replicate.rs index d079f7cf..8379927f 100644 --- a/lib/protoflow-blocks/src/blocks/flow/replicate.rs +++ b/lib/protoflow-blocks/src/blocks/flow/replicate.rs @@ -2,7 +2,7 @@ use crate::{FlowBlocks, StdioConfig, StdioError, StdioSystem, SysBlocks, System}; use protoflow_core::{ - types::Any, Block, BlockResult, BlockRuntime, InputPort, Message, OutputPort, + info, types::Any, Block, BlockResult, BlockRuntime, InputPort, Message, OutputPort, }; use protoflow_derive::Block; use simple_mermaid::mermaid; @@ -76,8 +76,7 @@ impl Block for Replicate { runtime.wait_for(&self.output_2)?; while let Some(message) = self.input.recv()? { - #[cfg(feature = "tracing")] - tracing::info!("Sending message"); + info!("Sending message"); self.output_1.send(&message)?; self.output_2.send(&message)?; } diff --git a/lib/protoflow-blocks/src/blocks/flow/sort.rs b/lib/protoflow-blocks/src/blocks/flow/sort.rs index a402a2af..5dbc7956 100644 --- a/lib/protoflow-blocks/src/blocks/flow/sort.rs +++ b/lib/protoflow-blocks/src/blocks/flow/sort.rs @@ -4,12 +4,11 @@ use core::cmp::Ordering; use crate::{StdioConfig, StdioError, StdioSystem, System}; use protoflow_core::{ - prelude::Vec, types::Any, Block, BlockResult, BlockRuntime, InputPort, Message, OutputPort, + error, info, prelude::Vec, types::Any, Block, BlockResult, BlockRuntime, InputPort, Message, + OutputPort, }; use protoflow_derive::Block; use simple_mermaid::mermaid; -#[cfg(feature = "tracing")] -use tracing::error; /// Sorts a single input message stream in ascending order. /// @@ -82,13 +81,11 @@ impl Block for Sort { self.messages.push(message); } - #[cfg(feature = "tracing")] - tracing::info!("Sorting messages"); + info!("Sorting messages"); self.messages.sort_by(|x, y| { if let Some(ordering) = x.partial_cmp(y) { ordering } else { - #[cfg(feature = "tracing")] error!("Incomparable values: {:?} and {:?}", x, y); Ordering::Equal } @@ -135,9 +132,7 @@ mod tests { fn run_sort_stdout() { use super::*; use crate::SysBlocks; - use protoflow_core::SystemBuilding; - #[cfg(feature = "tracing")] - use tracing::error; + use protoflow_core::{error, SystemBuilding}; if let Err(e) = System::run(|s| { let stdin = s.read_stdin(); @@ -147,7 +142,6 @@ mod tests { let stdout_1 = s.write_stdout(); s.connect(&sort.output, &stdout_1.input); }) { - #[cfg(feature = "tracing")] error!("{}", e) } } diff --git a/lib/protoflow-blocks/src/blocks/flow/split.rs b/lib/protoflow-blocks/src/blocks/flow/split.rs index d1a90334..8175325e 100644 --- a/lib/protoflow-blocks/src/blocks/flow/split.rs +++ b/lib/protoflow-blocks/src/blocks/flow/split.rs @@ -113,9 +113,7 @@ impl StdioSystem for Split { #[cfg(test)] mod split_tests { use crate::{CoreBlocks, FlowBlocks, SysBlocks, System}; - use protoflow_core::prelude::String; - #[cfg(feature = "tracing")] - use tracing::error; + use protoflow_core::{error, prelude::String}; extern crate std; #[test] @@ -147,7 +145,6 @@ mod split_tests { s.connect(&file.output, &write_file.path); s.connect(&split.output_2, &write_file.input); }) { - #[cfg(feature = "tracing")] error!("{}", e) } } @@ -168,7 +165,6 @@ mod split_tests { let stdout_2 = s.write_stdout(); s.connect(&split.output_2, &stdout_2.input); }) { - #[cfg(feature = "tracing")] error!("{}", e) } } diff --git a/lib/protoflow-blocks/src/blocks/sys/read_socket.rs b/lib/protoflow-blocks/src/blocks/sys/read_socket.rs index 37df1356..1a62505f 100644 --- a/lib/protoflow-blocks/src/blocks/sys/read_socket.rs +++ b/lib/protoflow-blocks/src/blocks/sys/read_socket.rs @@ -100,7 +100,6 @@ impl Block for ReadSocket { fn prepare(&mut self, _runtime: &dyn BlockRuntime) -> BlockResult { let listener = TcpListener::bind(&self.config.connection)?; *self.listener.lock().map_err(lock_error)? = Some(listener); - #[cfg(feature = "tracing")] info!("Server listening on {}", &self.config.connection); Ok(()) } @@ -115,7 +114,6 @@ impl Block for ReadSocket { .ok_or(BlockError::Other("Invalid TCP listener".into()))?; let (stream, addr) = listener.accept().map_err(|e| { - #[cfg(feature = "tracing")] error!("Failed to accept client connection: {}", e); BlockError::Other("Failed to accept client connection".into()) })?; @@ -125,7 +123,6 @@ impl Block for ReadSocket { if let Some(stream) = stream_guard.as_mut() { handle_client::<_>(stream, self.config.buffer_size, |message| { - #[cfg(feature = "tracing")] info!("Processing received message"); if self.output.is_connected() { self.output.send(message)?; @@ -133,7 +130,6 @@ impl Block for ReadSocket { Ok(()) }) .map_err(|e| { - #[cfg(feature = "tracing")] error!("Error handling client: {}", e); BlockError::Other("Error handling client".into()) })?; @@ -161,17 +157,14 @@ where let bytes_read = stream.read(&mut buffer)?; if bytes_read == 0 { - #[cfg(feature = "tracing")] info!("Client disconnected"); break; } let message = Bytes::copy_from_slice(&buffer[..bytes_read]); - #[cfg(feature = "tracing")] info!("Received message: {:?}", message); if let Err(e) = process_fn(&message) { - #[cfg(feature = "tracing")] error!("Failed to process message: {:?}", e); return Err(BlockError::Other("Failed to process message".into())); } @@ -221,7 +214,6 @@ pub mod read_socket_tests { )); s.connect(&read_socket.output, &std_out.input); }) { - #[cfg(feature = "tracing")] error!("{}", e) } } diff --git a/lib/protoflow-blocks/src/blocks/sys/write_socket.rs b/lib/protoflow-blocks/src/blocks/sys/write_socket.rs index 810ac3a4..c5802488 100644 --- a/lib/protoflow-blocks/src/blocks/sys/write_socket.rs +++ b/lib/protoflow-blocks/src/blocks/sys/write_socket.rs @@ -99,7 +99,6 @@ impl Block for WriteSocket { if stream_guard.is_none() { *stream_guard = Some(TcpStream::connect(&self.config.connection).map_err(|e| { - #[cfg(feature = "tracing")] error!("Failed to connect to {}: {}", &self.config.connection, e); BlockError::Other(format!( "Failed to connect to {}: {}", @@ -109,7 +108,6 @@ impl Block for WriteSocket { } let stream = stream_guard.as_mut().ok_or_else(|| { - #[cfg(feature = "tracing")] error!("Stream is not connected"); BlockError::Other("Stream is not connected".into()) })?; @@ -158,7 +156,6 @@ pub mod write_socket_tests { }); s.connect(&stdin.output, &write_socket.input); }) { - #[cfg(feature = "tracing")] error!("{}", e) } } From f1506ff185d99f829fc815a8e90a58020669f465 Mon Sep 17 00:00:00 2001 From: evren Date: Mon, 23 Dec 2024 10:54:08 +0300 Subject: [PATCH 23/24] fix doc of batch block --- lib/protoflow-blocks/src/blocks/flow/batch.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/protoflow-blocks/src/blocks/flow/batch.rs b/lib/protoflow-blocks/src/blocks/flow/batch.rs index 3a6025d5..502f3c61 100644 --- a/lib/protoflow-blocks/src/blocks/flow/batch.rs +++ b/lib/protoflow-blocks/src/blocks/flow/batch.rs @@ -27,7 +27,7 @@ use simple_mermaid::mermaid; /// # fn main() { /// System::build(|s| { /// let stdin = s.read_stdin(); -/// let batch = s.batch().batch(2); +/// let batch = s.batch(2); /// s.connect(&stdin.output, &batch.input); /// }); /// # } From d7df7973b924f3dd416d0d93e65f51f1aabf7991 Mon Sep 17 00:00:00 2001 From: evren Date: Mon, 23 Dec 2024 13:53:40 +0300 Subject: [PATCH 24/24] MapInto block --- README.md | 26 ++++- lib/protoflow-blocks/doc/flow/map_into.mmd | 11 +++ .../doc/flow/map_into.seq.mmd | 21 ++++ lib/protoflow-blocks/src/block_config.rs | 9 +- lib/protoflow-blocks/src/block_tag.rs | 4 + lib/protoflow-blocks/src/blocks/flow.rs | 18 ++++ .../src/blocks/flow/map_into.rs | 98 +++++++++++++++++++ lib/protoflow-blocks/src/lib.rs | 3 +- lib/protoflow-blocks/src/system.rs | 12 ++- 9 files changed, 192 insertions(+), 10 deletions(-) create mode 100644 lib/protoflow-blocks/doc/flow/map_into.mmd create mode 100644 lib/protoflow-blocks/doc/flow/map_into.seq.mmd create mode 100644 lib/protoflow-blocks/src/blocks/flow/map_into.rs diff --git a/README.md b/README.md index 81a919f1..2a360a50 100644 --- a/README.md +++ b/README.md @@ -130,7 +130,8 @@ The built-in blocks provided by Protoflow are listed below: | [`EncodeHex`] | Encodes a byte stream into hexadecimal form. | | [`EncodeJSON`] | Encodes messages into JSON format. | | [`Hash`] | Computes the cryptographic hash of a byte stream. | -| [`Merge`] | Merges multiple input message streams into a single output stream by interleaving messages as they arrive. | +| [`MapInto`] | Maps a message from one type to another via Into trait. | +| [`Merge`] | Merges multiple input message streams into a single output stream by interleaving messages as they arrive. | | [`Random`] | Generates and sends a random value. | | [`ReadDir`] | Reads file names from a file system directory. | | [`ReadEnv`] | Reads the value of an environment variable. | @@ -560,6 +561,28 @@ block-beta protoflow execute Hash algorithm=blake3 ``` +#### [`MapInto`] + +Maps a message from one type to another via Into trait. + +```mermaid +block-beta + columns 7 + Source space:2 MapInto space:2 Sink + Source-- "input" -->MapInto + MapInto-- "output" -->Sink + + classDef block height:48px,padding:8px; + classDef hidden visibility:none; + class MapInto block + class Source hidden + class Sink hidden +``` + +```bash +protoflow execute MapInto +``` + #### [`Merge`] Combines multiple input message streams into a single output stream by interleaving messages as they arrive. @@ -992,6 +1015,7 @@ To add a new block type implementation, make sure to examine and amend: [`EncodeHex`]: https://docs.rs/protoflow-blocks/latest/protoflow_blocks/struct.EncodeHex.html [`EncodeJSON`]: https://docs.rs/protoflow-blocks/latest/protoflow_blocks/struct.EncodeJson.html [`Hash`]: https://docs.rs/protoflow-blocks/latest/protoflow_blocks/struct.Hash.html +[`MapInto`]: https://docs.rs/protoflow-blocks/latest/protoflow_blocks/struct.MapInto.html [`Merge`]: https://docs.rs/protoflow-blocks/latest/protoflow_blocks/struct.Merge.html [`Random`]: https://docs.rs/protoflow-blocks/latest/protoflow_blocks/struct.Random.html [`ReadDir`]: https://docs.rs/protoflow-blocks/latest/protoflow_blocks/struct.ReadDir.html diff --git a/lib/protoflow-blocks/doc/flow/map_into.mmd b/lib/protoflow-blocks/doc/flow/map_into.mmd new file mode 100644 index 00000000..735bbcef --- /dev/null +++ b/lib/protoflow-blocks/doc/flow/map_into.mmd @@ -0,0 +1,11 @@ +block-beta + columns 7 + Source space:2 MapInto space:2 Sink + Source-- "input" -->MapInto + MapInto-- "output" -->Sink + + classDef block height:48px,padding:8px; + classDef hidden visibility:none; + class MapInto block + class Source hidden + class Sink hidden diff --git a/lib/protoflow-blocks/doc/flow/map_into.seq.mmd b/lib/protoflow-blocks/doc/flow/map_into.seq.mmd new file mode 100644 index 00000000..b9d2c99f --- /dev/null +++ b/lib/protoflow-blocks/doc/flow/map_into.seq.mmd @@ -0,0 +1,21 @@ +sequenceDiagram + autonumber + participant BlockA as Another block + participant MapInto.input as MapInto.input port + participant MapInto as MapInto block + participant MapInto.output as MapInto.output port + participant BlockB as Another block + + BlockA-->>MapInto: Connect + MapInto-->>BlockB: Connect + + loop MapInto process + BlockA->>MapInto: Message + MapInto->>MapInto: Transform + MapInto->>BlockB: Message + end + + BlockA-->>MapInto: Disconnect + MapInto-->>MapInto.input: Close + MapInto-->>MapInto.output: Close + MapInto-->>BlockB: Disconnect diff --git a/lib/protoflow-blocks/src/block_config.rs b/lib/protoflow-blocks/src/block_config.rs index 11955a67..d4c400e4 100644 --- a/lib/protoflow-blocks/src/block_config.rs +++ b/lib/protoflow-blocks/src/block_config.rs @@ -57,11 +57,10 @@ impl<'de> serde::Deserialize<'de> for BlockConfig { .unwrap() } - "Batch" | "Concat" | "Distinct" | "Merge" | "Replicate" | "Sort" | "Split" => { - FlowBlockConfig::deserialize(value.clone()) - .map(BlockConfig::Flow) - .unwrap() - } + "Batch" | "Concat" | "Distinct" | "MapInto" | "Merge" | "Replicate" | "Sort" + | "Split" => FlowBlockConfig::deserialize(value.clone()) + .map(BlockConfig::Flow) + .unwrap(), #[cfg(any( feature = "hash-blake3", diff --git a/lib/protoflow-blocks/src/block_tag.rs b/lib/protoflow-blocks/src/block_tag.rs index 61fdfd73..93bf614c 100644 --- a/lib/protoflow-blocks/src/block_tag.rs +++ b/lib/protoflow-blocks/src/block_tag.rs @@ -21,6 +21,7 @@ pub enum BlockTag { Batch, Concat, Distinct, + MapInto, Merge, Replicate, Sort, @@ -101,6 +102,7 @@ impl BlockTag { Encode => "Encode", EncodeHex => "EncodeHex", EncodeJson => "EncodeJSON", + MapInto => "MapInto", Merge => "Merge", #[cfg(feature = "std")] ReadDir => "ReadDir", @@ -159,6 +161,7 @@ impl FromStr for BlockTag { "Encode" => Encode, "EncodeHex" => EncodeHex, "EncodeJSON" => EncodeJson, + "MapInto" => MapInto, "Merge" => Merge, #[cfg(feature = "std")] "ReadDir" => ReadDir, @@ -228,6 +231,7 @@ impl BlockInstantiation for BlockTag { Encode => Box::new(super::Encode::::with_system(system, None)), EncodeHex => Box::new(super::EncodeHex::with_system(system)), EncodeJson => Box::new(super::EncodeJson::with_system(system)), + MapInto => Box::new(super::MapInto::::with_system(system)), Merge => Box::new(super::Merge::::with_system(system)), #[cfg(feature = "std")] ReadDir => Box::new(super::ReadDir::with_system(system)), diff --git a/lib/protoflow-blocks/src/blocks/flow.rs b/lib/protoflow-blocks/src/blocks/flow.rs index 8dc3e368..b676021b 100644 --- a/lib/protoflow-blocks/src/blocks/flow.rs +++ b/lib/protoflow-blocks/src/blocks/flow.rs @@ -14,6 +14,9 @@ pub mod flow { fn batch + 'static>(&mut self, batch_size: usize) -> Batch; fn concat + 'static>(&mut self) -> Concat; fn distinct + PartialEq + 'static>(&mut self) -> Distinct; + fn map_into + 'static, Output: Message + 'static>( + &mut self, + ) -> MapInto; fn merge + 'static>(&mut self) -> Merge; fn replicate + 'static>(&mut self) -> Replicate; fn sort + PartialOrd + 'static>(&mut self) -> Sort; @@ -26,6 +29,7 @@ pub mod flow { Batch, Concat, Distinct, + MapInto, Merge, Replicate, Sort, @@ -48,6 +52,10 @@ pub mod flow { input: InputPortName, output: OutputPortName, }, + MapInto { + input: InputPortName, + output: OutputPortName, + }, Merge { input_1: InputPortName, input_2: InputPortName, @@ -76,6 +84,7 @@ pub mod flow { Batch { .. } => "Batch", Concat { .. } => "Concat", Distinct { .. } => "Distinct", + MapInto { .. } => "MapInto", Merge { .. } => "Merge", Replicate { .. } => "Replicate", Sort { .. } => "Sort", @@ -97,6 +106,9 @@ pub mod flow { Distinct { output, .. } => { vec![("output", Some(output.clone()))] } + MapInto { output, .. } => { + vec![("output", Some(output.clone()))] + } Merge { output, .. } => { vec![("output", Some(output.clone()))] } @@ -137,6 +149,9 @@ pub mod flow { Distinct { .. } => { Box::new(super::Distinct::new(system.input_any(), system.output())) } + MapInto { .. } => { + Box::new(super::MapInto::new(system.input_any(), system.output_any())) + } Merge { .. } => Box::new(super::Merge::new( system.input_any(), system.input_any(), @@ -169,6 +184,9 @@ pub mod flow { mod distinct; pub use distinct::*; + mod map_into; + pub use map_into::*; + mod merge; pub use merge::*; diff --git a/lib/protoflow-blocks/src/blocks/flow/map_into.rs b/lib/protoflow-blocks/src/blocks/flow/map_into.rs new file mode 100644 index 00000000..b11190be --- /dev/null +++ b/lib/protoflow-blocks/src/blocks/flow/map_into.rs @@ -0,0 +1,98 @@ +// This is free and unencumbered software released into the public domain. + +use crate::{prelude::Bytes, StdioConfig, StdioError, StdioSystem, System}; +use protoflow_core::{info, Block, BlockResult, BlockRuntime, InputPort, Message, OutputPort}; +use protoflow_derive::Block; +use simple_mermaid::mermaid; + +/// A block to map a message from one type to another. +/// +/// # Block Diagram +#[doc = mermaid!("../../../doc/flow/map_into.mmd")] +/// +/// # Sequence Diagram +#[doc = mermaid!("../../../doc/flow/map_into.seq.mmd" framed)] +/// +/// # Examples +/// +/// ## Using the block in a system +/// +/// ```rust +/// # use protoflow_blocks::*; +/// # fn main() { +/// System::build(|s| { +/// // TODO +/// }); +/// # } +/// ``` +/// +#[derive(Block, Clone)] +pub struct MapInto, Output: Message> { + /// The input message stream. + #[input] + pub input: InputPort, + + /// The output message stream. + #[output] + pub output: OutputPort, +} + +impl, Output: Message> MapInto { + pub fn new(input: InputPort, output: OutputPort) -> Self { + Self::with_params(input, output) + } +} + +impl, Output: Message> MapInto { + pub fn with_params(input: InputPort, output: OutputPort) -> Self { + Self { input, output } + } +} + +impl + 'static, Output: Message + 'static> MapInto { + pub fn with_system(system: &System) -> Self { + use crate::SystemBuilding; + Self::with_params(system.input(), system.output()) + } +} + +impl, Output: Message> Block for MapInto { + fn execute(&mut self, _: &dyn BlockRuntime) -> BlockResult { + while let Some(input) = self.input.recv()? { + info!("Received input: {:?}", input); + let output: Output = Into::into(input); + info!("Sending output: {:?}", output); + self.output.send(&output)?; + } + + Ok(()) + } +} +#[cfg(feature = "std")] +impl, Output: Message> StdioSystem for MapInto { + fn build_system(config: StdioConfig) -> Result { + use crate::SystemBuilding; + + config.reject_any()?; + + Ok(System::build(|s| { + let stdin = config.read_stdin(s); + let map = s.block(MapInto::::with_system(s)); + s.connect(&stdin.output, &map.input); + })) + } +} + +#[cfg(test)] +mod tests { + use super::MapInto; + use crate::{System, SystemBuilding}; + + #[test] + fn instantiate_block() { + // Check that the block is constructible: + let _ = System::build(|s| { + let _ = s.block(MapInto::::with_params(s.input(), s.output())); + }); + } +} diff --git a/lib/protoflow-blocks/src/lib.rs b/lib/protoflow-blocks/src/lib.rs index 9cad0a93..7d44d869 100644 --- a/lib/protoflow-blocks/src/lib.rs +++ b/lib/protoflow-blocks/src/lib.rs @@ -53,7 +53,6 @@ pub fn build_stdio_system( use prelude::String; Ok(match system_name.as_ref() { // CoreBlocks - "Batch" => Batch::::build_system(config)?, "Buffer" => Buffer::::build_system(config)?, "Const" => Const::::build_system(config)?, "Count" => Count::::build_system(config)?, @@ -61,8 +60,10 @@ pub fn build_stdio_system( "Drop" => Drop::::build_system(config)?, "Random" => Random::::build_system(config)?, // FlowBlocks + "Batch" => Batch::::build_system(config)?, "Concat" => Concat::::build_system(config)?, "Distinct" => Distinct::::build_system(config)?, + "MapInto" => MapInto::::build_system(config)?, "Merge" => Merge::::build_system(config)?, "Replicate" => Replicate::::build_system(config)?, "Sort" => Sort::::build_system(config)?, diff --git a/lib/protoflow-blocks/src/system.rs b/lib/protoflow-blocks/src/system.rs index b4cb4b26..633bb065 100644 --- a/lib/protoflow-blocks/src/system.rs +++ b/lib/protoflow-blocks/src/system.rs @@ -7,9 +7,9 @@ use crate::{ types::{DelayType, Encoding}, AllBlocks, Batch, Buffer, Concat, ConcatStrings, Const, CoreBlocks, Count, Decode, DecodeCsv, DecodeHex, DecodeJson, Delay, Distinct, Drop, Encode, EncodeCsv, EncodeHex, EncodeJson, - FlowBlocks, HashBlocks, IoBlocks, MathBlocks, Merge, Random, ReadDir, ReadEnv, ReadFile, - ReadStdin, Replicate, Sort, Split, SplitString, SysBlocks, TextBlocks, WriteFile, WriteStderr, - WriteStdout, + FlowBlocks, HashBlocks, IoBlocks, MapInto, MathBlocks, Merge, Random, ReadDir, ReadEnv, + ReadFile, ReadStdin, Replicate, Sort, Split, SplitString, SysBlocks, TextBlocks, WriteFile, + WriteStderr, WriteStdout, }; #[cfg(all(feature = "std", feature = "serde"))] use crate::{ReadSocket, WriteSocket}; @@ -192,6 +192,12 @@ impl FlowBlocks for System { self.0.block(Distinct::::with_system(self)) } + fn map_into + 'static, Output: Message + 'static>( + &mut self, + ) -> MapInto { + self.0.block(MapInto::::with_system(self)) + } + fn merge + 'static>(&mut self) -> Merge { self.0.block(Merge::::with_system(self)) }