Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 15 additions & 5 deletions flow/record/tools/rdump.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@

import flow.record.adapter
from flow.record import RecordWriter, iter_timestamped_records, record_stream
from flow.record.exceptions import RecordAdapterNotFound
from flow.record.selector import make_selector
from flow.record.stream import RecordFieldRewriter
from flow.record.utils import LOGGING_TRACE_LEVEL, catch_sigpipe
Expand Down Expand Up @@ -143,6 +144,11 @@ def main(argv: list[str] | None = None) -> int:
action="store_true",
help="Show count of processed records",
)
output.add_argument(
"--ignore-empty-record-stream",
action="store_true",
help="Exit successfully if no records can be read",
)

advanced = parser.add_argument_group("advanced")
advanced.add_argument(
Expand Down Expand Up @@ -315,13 +321,17 @@ def main(argv: list[str] | None = None) -> int:
record_writer.write(rec)

except Exception as e:
print_error(e)
if args.ignore_empty_record_stream and count == 0 and isinstance(e, RecordAdapterNotFound):
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why also RecordAdapterNotFound in the condition?

log.warning("No records were read from the input stream")
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If someone would use a selector that yields no records this warning would not be technically correct. As records have ben read from input stream but it yielded no records due to the selector.

I was thinking about adding extra return data to record_stream for how many records have been read (also for improved progress bar), but the selector is usually applied in the adapter so record_stream has no telemetry on this. Needs some more thought on how this could be achieved though and better for another PR.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

processed records are now tracked via an AppContext, see #184

ret = 0
else:
print_error(e)

# Prevent throwing an exception twice when deconstructing the record writer.
if hasattr(record_writer, "exception") and record_writer.exception is e:
record_writer.exception = None
# Prevent throwing an exception twice when deconstructing the record writer.
if hasattr(record_writer, "exception") and record_writer.exception is e:
record_writer.exception = None

ret = 1
ret = 1

finally:
if record_writer:
Expand Down
22 changes: 22 additions & 0 deletions tests/test_rdump.py
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,28 @@ def test_rdump_pipe(tmp_path: Path) -> None:
assert len(records) == 3
assert {r.count for r in records} == {1, 3, 9}

# Test if rdump exits with exit code 1 when no records can be read from stdin
p1 = subprocess.Popen(["echo", ""], stdout=subprocess.PIPE)
p2 = subprocess.Popen(["rdump", "-"], stdin=p1.stdout, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
stdout, stderr = p2.communicate()

assert p2.returncode == 1
assert b"rdump encountered a fatal error: Could not find adapter for file-like object" in stderr

# Test if rdump exits with exit code 0 when no records can be read from stdin
# and --ignore-empty-record-stream is set
p1 = subprocess.Popen(["echo", ""], stdout=subprocess.PIPE)
p2 = subprocess.Popen(
["rdump", "-v", "-", "--ignore-empty-record-stream"],
stdin=p1.stdout,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
stdout, stderr = p2.communicate()

assert p2.returncode == 0
assert b"No records were read from the input stream" in stderr


def test_rdump_format_template(tmp_path: Path) -> None:
TestRecord = RecordDescriptor(
Expand Down
Loading