Allow user to see verifier logs using flags

README.md

@@ -68,6 +68,10 @@ with the `BPF_F_XDP_HAS_FRAGS` flag.
 * Display captured packets:
 `sudo xdpcap /path/to/pinned/map - "tcp and port 80" | sudo tcpdump -r -`
 
+if you wish to see the verifier logs in cases where the program fails the checks
+you can use the `-print-verifier-logs` flag. You can control the size of the buffer
+for the verifier logs using the flag `-verifier-log-size`.
+
 
 ## Limitations
 

cmd/xdpcap/filter.go

@@ -44,17 +44,17 @@ type filter struct {
 }
 
 // newFilter creates a filter from a tcpdump / libpcap filter expression
-func newFilter(hookMapPath string, opts filterOpts) (*filter, error) {
+func newFilter(hookMapPath string, opts filterOpts, programOpts ebpf.ProgramOptions) (*filter, error) {
 	hookMap, err := ebpf.LoadPinnedMap(hookMapPath, nil)
 	if err != nil {
 		return nil, errors.Wrapf(err, "loading hook map")
 	}
 
-	return newFilterWithMap(hookMap, opts)
+	return newFilterWithMap(hookMap, opts, programOpts)
 }
 
 // newFilterWithMap creates a filter from a tcpdump / libpcap filter expression
-func newFilterWithMap(hookMap *ebpf.Map, opts filterOpts) (*filter, error) {
+func newFilterWithMap(hookMap *ebpf.Map, opts filterOpts, programOpts ebpf.ProgramOptions) (*filter, error) {
 	if len(opts.filter) == 0 {
 		return nil, errors.New("at least one filter cBPF instruction required")
 	}
@@ -91,7 +91,7 @@ func newFilterWithMap(hookMap *ebpf.Map, opts filterOpts) (*filter, error) {
 
 	xdpFragsMode := false
 	for i, action := range opts.actions {
-		program, err := newProgram(opts.filter, action, perfMap, xdpFragsMode)
+		program, err := newProgram(opts.filter, action, perfMap, xdpFragsMode, programOpts)
 		if err != nil {
 			return nil, errors.Wrapf(err, "loading filter program for %v", action)
 		}
@@ -105,7 +105,7 @@ func newFilterWithMap(hookMap *ebpf.Map, opts filterOpts) (*filter, error) {
 			xdpFragsMode = true
 
 			var programErr error
-			if program, programErr = newProgram(opts.filter, action, perfMap, xdpFragsMode); programErr != nil {
+			if program, programErr = newProgram(opts.filter, action, perfMap, xdpFragsMode, programOpts); programErr != nil {
 				return nil, errors.Wrapf(programErr, "loading filter program in XDP frags mode for %v", action)
 			}
 
@@ -145,7 +145,6 @@ func attachProg(hookMap *ebpf.Map, fd int, action xdpAction) error {
 	if err != nil {
 		return errors.Wrap(err, "attaching filter programs")
 	}
-
 	return nil
 }
 

cmd/xdpcap/filter_test.go

@@ -40,7 +40,7 @@ func TestMain(m *testing.M) {
 }
 
 func TestMissingFilter(t *testing.T) {
-	_, err := newFilterWithMap(hookMap(t, 1), testOpts())
+	_, err := newFilterWithMap(hookMap(t, 1), testOpts(), ebpf.ProgramOptions{})
 	if err == nil {
 		t.Fatal("empty filter accepted")
 	}
@@ -85,7 +85,7 @@ func TestFilterProgramForAllModes(t *testing.T) {
 
 			opts := testOpts(bpf.RetConstant{Val: 0})
 			opts.actions = []xdpAction{xdpPass}
-			filter, err := newFilterWithMap(hookMap, opts)
+			filter, err := newFilterWithMap(hookMap, opts, ebpf.ProgramOptions{})
 			if err != nil {
 				t.Fatal(err)
 			}
@@ -116,7 +116,7 @@ func TestAllActions(t *testing.T) {
 	opts.actions = []xdpAction{}
 
 	// progs with actions from 0-9. Only 0-3 are used currently.
-	filter, err := newFilterWithMap(hookMap(t, 10), opts)
+	filter, err := newFilterWithMap(hookMap(t, 10), opts, ebpf.ProgramOptions{})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -298,7 +298,7 @@ func hookMap(t *testing.T, entries int) *ebpf.Map {
 func mustNew(t *testing.T, opts filterOpts) *filter {
 	t.Helper()
 
-	filter, err := newFilterWithMap(hookMap(t, len(opts.actions)), opts)
+	filter, err := newFilterWithMap(hookMap(t, len(opts.actions)), opts, ebpf.ProgramOptions{})
 	if err != nil {
 		t.Fatal(err)
 	}

cmd/xdpcap/flags.go

@@ -9,6 +9,7 @@ import (
 	"strconv"
 	"strings"
 
+	"github.com/cilium/ebpf"
 	"github.com/cloudflare/xdpcap/internal"
 
 	"github.com/google/gopacket/layers"
@@ -181,6 +182,9 @@ type flags struct {
 	// Filter provided as input. Not in any particular format, for metadata / debugging only.
 	filterExpr string
 	filterOpts filterOpts
+
+	printVerifierLogs bool
+	verifierLogSize   int
 }
 
 // parseFlags creates the flags, and attempts to parse args.
@@ -198,6 +202,8 @@ func parseFlags(name string, args []string) (flags, error) {
 	flags.IntVar(&flags.filterOpts.perfWatermark, "watermark", 1, "Perf watermark (`bytes`). Must be < buffer.")
 	flags.BoolVar(&flags.quiet, "q", false, "Don't print statistics")
 	flags.BoolVar(&flags.flush, "flush", false, "Flush pcap data written to <output> for every packet received")
+	flags.BoolVar(&flags.printVerifierLogs, "print-verifier-logs", false, "If the verifier rejects the eBPF program, dump the logs")
+	flags.IntVar(&flags.verifierLogSize, "verifier-log-size", ebpf.DefaultVerifierLogSize, "Size of buffer to use for the verifier logs")
 
 	flags.filterOpts.actions = []xdpAction{}
 	flags.Var((*actionsFlag)(&flags.filterOpts.actions), "actions", fmt.Sprintf("XDP `actions` to capture packets for. Comma separated list of names (%v) or enum values (default all actions exposed by the <debug map>)", xdpActions))

cmd/xdpcap/flags_test.go

@@ -8,6 +8,7 @@ import (
 	"strings"
 	"testing"
 
+	"github.com/cilium/ebpf"
 	"github.com/cloudflare/xdpcap/internal"
 
 	"github.com/google/gopacket/layers"
@@ -17,18 +18,18 @@ import (
 func TestRequiredArgs(t *testing.T) {
 	output := tempOutput(t)
 
-	flags, err := parseFlags("", []string{})
+	_, err := parseFlags("", []string{})
 	if err == nil {
 		t.Fatal("missing main args")
 	}
 
-	flags, err = parseFlags("", []string{"foo"})
+	_, err = parseFlags("", []string{"foo"})
 	if err == nil {
 		t.Fatal("missing main args")
 	}
 
 	// Two args - empty filter
-	flags, err = parseFlags("", []string{"foo", output})
+	flags, err := parseFlags("", []string{"foo", output})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -253,12 +254,13 @@ func tempOutput(t *testing.T) string {
 
 func defaultFlags(mapPath string) flags {
 	return flags{
-		mapPath:    mapPath,
-		pcapFile:   nil,
-		quiet:      false,
-		flush:      false,
-		linkType:   layers.LinkTypeEthernet,
-		filterExpr: "",
+		mapPath:         mapPath,
+		pcapFile:        nil,
+		quiet:           false,
+		flush:           false,
+		linkType:        layers.LinkTypeEthernet,
+		filterExpr:      "",
+		verifierLogSize: ebpf.DefaultVerifierLogSize,
 		filterOpts: filterOpts{
 			perfPerCPUBuffer: 8192,
 			perfWatermark:    1,

cmd/xdpcap/main.go

@@ -27,6 +27,7 @@ import (
 	"syscall"
 	"time"
 
+	"github.com/cilium/ebpf"
 	"github.com/google/gopacket"
 	"github.com/google/gopacket/layers"
 	"github.com/google/gopacket/pcapgo"
@@ -48,7 +49,12 @@ func main() {
 
 	err = capture(flags)
 	if err != nil {
-		fmt.Fprintln(os.Stderr, "Error:", err)
+		var verifierErr *ebpf.VerifierError
+		if errors.As(err, &verifierErr) && flags.printVerifierLogs {
+			fmt.Printf("Verifier error: %+v\n", verifierErr)
+		} else {
+			fmt.Fprintln(os.Stderr, "Error:", err)
+		}
 		os.Exit(2)
 	}
 }
@@ -64,7 +70,12 @@ func capture(flags flags) error {
 	sigs := make(chan os.Signal, 1)
 	signal.Notify(sigs, os.Interrupt, syscall.SIGTERM)
 
-	filter, err := newFilter(flags.mapPath, flags.filterOpts)
+	programOpts := ebpf.ProgramOptions{
+		LogSize:     flags.verifierLogSize,
+		LogDisabled: !flags.printVerifierLogs,
+		LogLevel:    ebpf.LogLevelInstruction,
+	}
+	filter, err := newFilter(flags.mapPath, flags.filterOpts, programOpts)
 	if err != nil {
 		return errors.Wrap(err, "creating filter")
 	}

cmd/xdpcap/program.go

@@ -43,7 +43,7 @@ type program struct {
 }
 
 // newProgram builds an eBPF program that copies packets matching a cBPF program to userspace via perf
-func newProgram(filter []bpf.Instruction, action xdpAction, perfMap *ebpf.Map, xdpFragsMode bool) (*program, error) {
+func newProgram(filter []bpf.Instruction, action xdpAction, perfMap *ebpf.Map, xdpFragsMode bool, opts ebpf.ProgramOptions) (*program, error) {
 	metricsMap, err := ebpf.NewMap(&metricsSpec)
 	if err != nil {
 		return nil, errors.Wrap(err, "creating metrics map")
@@ -172,7 +172,7 @@ func newProgram(filter []bpf.Instruction, action xdpAction, perfMap *ebpf.Map, x
 		progSpec.Flags = progSpec.Flags | unix.BPF_F_XDP_HAS_FRAGS
 	}
 
-	prog, err := ebpf.NewProgram(progSpec)
+	prog, err := ebpf.NewProgramWithOptions(progSpec, opts)
 	if err != nil {
 		return nil, errors.Wrap(err, "loading filter")
 	}