Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -150,6 +150,13 @@ tcpdump -w - -c 1000 | dsct read -
tcpdump -w - -i eth0 udp port 53 | dsct read - -f dns
```

Include the original packet bytes (link-layer included) as a hex string under
`raw_bytes` for downstream parsing or reconstruction:

```bash
dsct read capture.pcap --raw-bytes --count 1
```

Inspect available fields and schemas:

```bash
Expand Down
12 changes: 12 additions & 0 deletions benches/json_escape.rs
Original file line number Diff line number Diff line change
Expand Up @@ -152,6 +152,7 @@ fn bench_json_escape(c: &mut Criterion) {
black_box(&tcp_buf),
black_box(tcp_data),
None,
false,
)
.unwrap();
black_box(&buf);
Expand All @@ -168,6 +169,7 @@ fn bench_json_escape(c: &mut Criterion) {
black_box(&tcp_buf),
black_box(tcp_data),
None,
false,
)
.unwrap();
black_box(&buf);
Expand All @@ -186,6 +188,7 @@ fn bench_json_escape(c: &mut Criterion) {
black_box(&tcp_buf),
black_box(tcp_data),
None,
false,
)
.unwrap();
bw.flush().unwrap();
Expand All @@ -204,6 +207,7 @@ fn bench_json_escape(c: &mut Criterion) {
black_box(&tcp_buf),
black_box(tcp_data),
None,
false,
)
.unwrap();
let mut ew = JsonEscapeWriter::new(&mut buf);
Expand All @@ -228,6 +232,7 @@ fn bench_json_escape(c: &mut Criterion) {
black_box(&dns_buf),
black_box(dns_data),
None,
false,
)
.unwrap();
black_box(&buf);
Expand All @@ -244,6 +249,7 @@ fn bench_json_escape(c: &mut Criterion) {
black_box(&dns_buf),
black_box(dns_data),
None,
false,
)
.unwrap();
black_box(&buf);
Expand All @@ -262,6 +268,7 @@ fn bench_json_escape(c: &mut Criterion) {
black_box(&dns_buf),
black_box(dns_data),
None,
false,
)
.unwrap();
bw.flush().unwrap();
Expand All @@ -280,6 +287,7 @@ fn bench_json_escape(c: &mut Criterion) {
black_box(&dns_buf),
black_box(dns_data),
None,
false,
)
.unwrap();
let mut ew = JsonEscapeWriter::new(&mut buf);
Expand All @@ -306,6 +314,7 @@ fn bench_json_escape(c: &mut Criterion) {
black_box(&dissect_buf),
black_box(&tp.raw),
None,
false,
)
.unwrap();
buf.push(b'\n');
Expand All @@ -330,6 +339,7 @@ fn bench_json_escape(c: &mut Criterion) {
black_box(&dissect_buf),
black_box(&tp.raw),
None,
false,
)
.unwrap();
ew.write_all(&pkt_buf).unwrap();
Expand All @@ -352,6 +362,7 @@ fn bench_json_escape(c: &mut Criterion) {
&tcp_buf,
&test_packets[0].raw,
None,
false,
)
.unwrap();
sample_json.push(b'\n');
Expand All @@ -361,6 +372,7 @@ fn bench_json_escape(c: &mut Criterion) {
&dns_buf,
&test_packets[1].raw,
None,
false,
)
.unwrap();
sample_json.push(b'\n');
Expand Down
7 changes: 7 additions & 0 deletions src/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -111,6 +111,11 @@ struct ReadOptions {
/// - `0x1234:aes-256-cbc:0xKEY:hmac-sha1-96:0xKEY`
#[arg(long = "esp-sa", num_args = 1)]
esp_sa: Vec<String>,

/// Include the original packet bytes (link-layer included) as a
/// lowercase hex string under the `raw_bytes` field of each record.
#[arg(long)]
raw_bytes: bool,
}

/// Options for the `dsct stats` command.
Expand Down Expand Up @@ -303,6 +308,7 @@ fn cmd_read(opts: ReadOptions) -> Result<()> {
progress,
decode_as: decode_as_args,
esp_sa: esp_sa_args,
raw_bytes,
} = opts;
// Resolve effective count: explicit --count, default limit, or unlimited.
let (count, is_default_limit) = if no_limit {
Expand Down Expand Up @@ -428,6 +434,7 @@ fn cmd_read(opts: ReadOptions) -> Result<()> {
dissect_buf,
data,
field_config.as_ref(),
raw_bytes,
)?;
pkt_buf.push(b'\n');
writer.write_all(&pkt_buf)?;
Expand Down
18 changes: 18 additions & 0 deletions src/mcp/raw_mcp.rs
Original file line number Diff line number Diff line change
Expand Up @@ -272,6 +272,11 @@ fn read_packets_schema() -> Value {
"default": false,
"description": "Show all fields including low-level details (checksums, header lengths, etc.)."
},
"raw_bytes": {
"type": "boolean",
"default": false,
"description": "Include the original packet bytes (link-layer included) as a lowercase hex string under the `raw_bytes` field of each record."
},
"sample_rate": {
"type": "integer",
"minimum": 1,
Expand Down Expand Up @@ -495,6 +500,10 @@ fn handle_read_packets_streaming(
.get("verbose")
.and_then(Value::as_bool)
.unwrap_or(false);
let raw_bytes = arguments
.get("raw_bytes")
.and_then(Value::as_bool)
.unwrap_or(false);
let field_config = if verbose {
None
} else {
Expand Down Expand Up @@ -624,6 +633,7 @@ fn handle_read_packets_streaming(
dissect_buf,
data,
field_config.as_ref(),
raw_bytes,
)?;
w.write_all(&pkt_buf)?;
packets_written += 1;
Expand Down Expand Up @@ -960,6 +970,14 @@ mod tests {
assert_eq!(verbose["default"], false);
}

#[test]
fn read_packets_schema_has_raw_bytes() {
let schema = read_packets_schema();
let raw = &schema["properties"]["raw_bytes"];
assert_eq!(raw["type"], "boolean");
assert_eq!(raw["default"], false);
}

#[test]
fn get_stats_schema_has_esp_sa() {
let schema = get_stats_schema();
Expand Down
13 changes: 13 additions & 0 deletions src/schema.rs
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,10 @@ pub fn read_schema() -> serde_json::Value {
}
}
}
},
"raw_bytes": {
"type": "string",
"description": "Original packet bytes (link-layer included) as a lowercase hex string. Present only when --raw-bytes is specified."
}
}
})
Expand Down Expand Up @@ -214,6 +218,15 @@ mod tests {
assert!(required.iter().any(|v| v == "layers"));
}

#[test]
fn read_schema_has_optional_raw_bytes_property() {
let schema = read_schema();
let raw = &schema["properties"]["raw_bytes"];
assert_eq!(raw["type"], "string");
let required = schema["required"].as_array().unwrap();
assert!(!required.iter().any(|v| v == "raw_bytes"));
}

#[test]
fn stats_schema_has_required_fields() {
let schema = stats_schema();
Expand Down
40 changes: 26 additions & 14 deletions src/serialize.rs
Original file line number Diff line number Diff line change
Expand Up @@ -155,6 +155,17 @@ fn write_timestamp_to<W: Write>(w: &mut W, secs: u64, usecs: u32) -> std::io::Re
// Streaming JSON write — zero-allocation packet serialization via DissectBuffer.
// ---------------------------------------------------------------------------

/// Write `bytes` as a JSON string of lowercase hex digits, including the
/// enclosing double quotes.
fn write_hex_string<W: Write>(w: &mut W, bytes: &[u8]) -> Result<()> {
w.write_all(b"\"")?;
for byte in bytes {
write!(w, "{byte:02x}")?;
}
w.write_all(b"\"")?;
Ok(())
}

// ---------------------------------------------------------------------------

/// Write a [`FieldValue`] as a JSON token to `w`.
Expand Down Expand Up @@ -206,20 +217,10 @@ fn write_raw_field_value_json<W: Write>(
write!(w, "\"{addr}\"")?;
}
FieldValue::MacAddr(m) => write!(w, "\"{m}\"")?,
FieldValue::Bytes(b) => {
write!(w, "\"")?;
for byte in *b {
write!(w, "{byte:02x}")?;
}
write!(w, "\"")?;
}
FieldValue::Bytes(b) => write_hex_string(w, b)?,
FieldValue::Scratch(range) => {
let scratch_bytes = &buf.scratch()[range.start as usize..range.end as usize];
write!(w, "\"")?;
for byte in scratch_bytes {
write!(w, "{byte:02x}")?;
}
write!(w, "\"")?;
write_hex_string(w, scratch_bytes)?;
}
FieldValue::Array(_) | FieldValue::Object(_) => {
// Container fields should be handled by write_field_json;
Expand Down Expand Up @@ -437,12 +438,16 @@ fn write_layer_fields<W: Write>(
///
/// Uses the flat [`DissectBuffer`] API — no intermediate serde structures
/// are allocated.
///
/// When `raw_bytes` is `true`, a trailing `"raw_bytes":"<hex>"` field is
/// appended with the original packet bytes (including link-layer headers).
pub fn write_packet_json<W: Write>(
w: &mut W,
meta: &PacketMeta,
buf: &DissectBuffer<'_>,
data: &[u8],
field_config: Option<&FieldConfig>,
raw_bytes: bool,
) -> Result<()> {
// number
write!(w, "{{\"number\":{},\"timestamp\":\"", meta.number)?;
Expand Down Expand Up @@ -473,7 +478,14 @@ pub fn write_packet_json<W: Write>(
write_layer_fields(w, layer, buf, data, field_config)?;
write!(w, "}}}}")?; // close fields, close layer
}
write!(w, "]}}")?; // close layers, close packet
// close layers
w.write_all(b"]")?;
if raw_bytes {
w.write_all(b",\"raw_bytes\":")?;
write_hex_string(w, data)?;
}
// close packet
w.write_all(b"}")?;
Ok(())
}

Expand Down Expand Up @@ -605,7 +617,7 @@ mod tests {
field_config: Option<&FieldConfig>,
) -> serde_json::Value {
let mut out = Vec::new();
write_packet_json(&mut out, meta, buf, data, field_config).unwrap();
write_packet_json(&mut out, meta, buf, data, field_config, false).unwrap();
serde_json::from_slice(&out).unwrap()
}

Expand Down
34 changes: 34 additions & 0 deletions tests/cli_output_snapshot_test.rs
Original file line number Diff line number Diff line change
Expand Up @@ -171,6 +171,40 @@ fn read_verbose_adds_fields_to_ipv4_layer() {
);
}

#[test]
fn read_raw_bytes_appends_field_at_end() {
let tmp = write_pcap(1);

let output = Command::cargo_bin("dsct")
.unwrap()
.args(["read", "--raw-bytes", tmp.path().to_str().unwrap()])
.output()
.unwrap();
assert!(output.status.success());

let stdout = String::from_utf8(output.stdout).unwrap();
let first_line = stdout.lines().next().expect("at least one JSONL line");
let value: Value = serde_json::from_str(first_line).unwrap();

// raw_bytes must be appended after `layers`, preserving the existing
// top-level key order.
let expected = [
"number",
"timestamp",
"length",
"original_length",
"stack",
"layers",
"raw_bytes",
];
assert_eq!(
object_keys(&value),
expected,
"raw_bytes must be appended at the end of the record"
);
assert!(value["raw_bytes"].is_string());
}

// ---------------------------------------------------------------------------
// `dsct stats` — StatsOutput schema
// ---------------------------------------------------------------------------
Expand Down
55 changes: 55 additions & 0 deletions tests/cli_read_test.rs
Original file line number Diff line number Diff line change
Expand Up @@ -890,3 +890,58 @@ fn esp_null_decoded_without_sa() {
"encrypted_data should not be emitted when auto-decoding succeeds"
);
}

#[test]
fn read_raw_bytes_emits_hex_matching_packet() {
let tmp = write_pcap(1);

let output = Command::cargo_bin("dsct")
.unwrap()
.args(["read", "--raw-bytes", tmp.path().to_str().unwrap()])
.output()
.unwrap();

assert!(output.status.success());

let stdout = String::from_utf8(output.stdout).unwrap();
let line = stdout.trim().lines().next().unwrap();
let v: serde_json::Value = serde_json::from_str(line).unwrap();
let raw = v["raw_bytes"].as_str().expect("raw_bytes must be a string");

let length = v["length"].as_u64().unwrap() as usize;
assert_eq!(raw.len(), length * 2, "hex length must be 2 * length");
assert!(
raw.chars()
.all(|c| c.is_ascii_hexdigit() && !c.is_ascii_uppercase()),
"raw_bytes must be lowercase hex"
);

// The fixed Ethernet/IPv4/UDP packet built by build_pcap starts with the
// broadcast destination MAC and has a well-known byte layout. Verify a
// prefix exact match to ensure bytes are passed through unchanged.
let expected_prefix = "ffffffffffff00112233445508004500001c";
assert!(
raw.starts_with(expected_prefix),
"unexpected raw_bytes prefix: {raw}"
);
}

#[test]
fn read_without_raw_bytes_omits_field() {
let tmp = write_pcap(1);

let output = Command::cargo_bin("dsct")
.unwrap()
.args(["read", tmp.path().to_str().unwrap()])
.output()
.unwrap();

assert!(output.status.success());
let stdout = String::from_utf8(output.stdout).unwrap();
let line = stdout.trim().lines().next().unwrap();
let v: serde_json::Value = serde_json::from_str(line).unwrap();
assert!(
v.get("raw_bytes").is_none(),
"raw_bytes must be absent when --raw-bytes is not set"
);
}
Loading