From 6a5aae95dd19d29307d1dc0ec1dfc149473ccecf Mon Sep 17 00:00:00 2001 From: halfrost Date: Wed, 26 Oct 2022 09:17:06 -0700 Subject: [PATCH 1/2] Add ebpf --- ebpf/ebpf-kernel/bpf_bpfeb.go | 119 ++++++++++++++++++++++++++++ ebpf/ebpf-kernel/bpf_bpfel.go | 121 ++++++++++++++++++++++++++++ ebpf/ebpf-kernel/main.go | 144 ++++++++++++++++++++++++++++++++++ ebpf/ebpf-kernel/tcprtt.c | 93 ++++++++++++++++++++++ ebpf/ebpf-kernel/xdp.c | 65 +++++++++++++++ ebpf/ebpf-user/main.go | 32 ++++++++ 6 files changed, 574 insertions(+) create mode 100644 ebpf/ebpf-kernel/bpf_bpfeb.go create mode 100644 ebpf/ebpf-kernel/bpf_bpfel.go create mode 100644 ebpf/ebpf-kernel/main.go create mode 100644 ebpf/ebpf-kernel/tcprtt.c create mode 100644 ebpf/ebpf-kernel/xdp.c create mode 100644 ebpf/ebpf-user/main.go diff --git a/ebpf/ebpf-kernel/bpf_bpfeb.go b/ebpf/ebpf-kernel/bpf_bpfeb.go new file mode 100644 index 0000000..c73697a --- /dev/null +++ b/ebpf/ebpf-kernel/bpf_bpfeb.go @@ -0,0 +1,119 @@ +// Code generated by bpf2go; DO NOT EDIT. +//go:build arm64be || armbe || mips || mips64 || mips64p32 || ppc64 || s390 || s390x || sparc || sparc64 +// +build arm64be armbe mips mips64 mips64p32 ppc64 s390 s390x sparc sparc64 + +package main + +import ( + "bytes" + _ "embed" + "fmt" + "io" + + "github.com/cilium/ebpf" +) + +// loadBpf returns the embedded CollectionSpec for bpf. +func loadBpf() (*ebpf.CollectionSpec, error) { + reader := bytes.NewReader(_BpfBytes) + spec, err := ebpf.LoadCollectionSpecFromReader(reader) + if err != nil { + return nil, fmt.Errorf("can't load bpf: %w", err) + } + + return spec, err +} + +// loadBpfObjects loads bpf and converts it into a struct. +// +// The following types are suitable as obj argument: +// +// *bpfObjects +// *bpfPrograms +// *bpfMaps +// +// See ebpf.CollectionSpec.LoadAndAssign documentation for details. +func loadBpfObjects(obj interface{}, opts *ebpf.CollectionOptions) error { + spec, err := loadBpf() + if err != nil { + return err + } + + return spec.LoadAndAssign(obj, opts) +} + +// bpfSpecs contains maps and programs before they are loaded into the kernel. +// +// It can be passed ebpf.CollectionSpec.Assign. +type bpfSpecs struct { + bpfProgramSpecs + bpfMapSpecs +} + +// bpfSpecs contains programs before they are loaded into the kernel. +// +// It can be passed ebpf.CollectionSpec.Assign. +type bpfProgramSpecs struct { + XdpProgFunc *ebpf.ProgramSpec `ebpf:"xdp_prog_func"` +} + +// bpfMapSpecs contains maps before they are loaded into the kernel. +// +// It can be passed ebpf.CollectionSpec.Assign. +type bpfMapSpecs struct { + XdpStatsMap *ebpf.MapSpec `ebpf:"xdp_stats_map"` +} + +// bpfObjects contains all objects after they have been loaded into the kernel. +// +// It can be passed to loadBpfObjects or ebpf.CollectionSpec.LoadAndAssign. +type bpfObjects struct { + bpfPrograms + bpfMaps +} + +func (o *bpfObjects) Close() error { + return _BpfClose( + &o.bpfPrograms, + &o.bpfMaps, + ) +} + +// bpfMaps contains all maps after they have been loaded into the kernel. +// +// It can be passed to loadBpfObjects or ebpf.CollectionSpec.LoadAndAssign. +type bpfMaps struct { + XdpStatsMap *ebpf.Map `ebpf:"xdp_stats_map"` +} + +func (m *bpfMaps) Close() error { + return _BpfClose( + m.XdpStatsMap, + ) +} + +// bpfPrograms contains all programs after they have been loaded into the kernel. +// +// It can be passed to loadBpfObjects or ebpf.CollectionSpec.LoadAndAssign. +type bpfPrograms struct { + XdpProgFunc *ebpf.Program `ebpf:"xdp_prog_func"` +} + +func (p *bpfPrograms) Close() error { + return _BpfClose( + p.XdpProgFunc, + ) +} + +func _BpfClose(closers ...io.Closer) error { + for _, closer := range closers { + if err := closer.Close(); err != nil { + return err + } + } + return nil +} + +// Do not access this directly. +//go:embed bpf_bpfeb.o +var _BpfBytes []byte \ No newline at end of file diff --git a/ebpf/ebpf-kernel/bpf_bpfel.go b/ebpf/ebpf-kernel/bpf_bpfel.go new file mode 100644 index 0000000..f391329 --- /dev/null +++ b/ebpf/ebpf-kernel/bpf_bpfel.go @@ -0,0 +1,121 @@ + + +// Code generated by bpf2go; DO NOT EDIT. +//go:build 386 || amd64 || amd64p32 || arm || arm64 || mips64le || mips64p32le || mipsle || ppc64le || riscv64 +// +build 386 amd64 amd64p32 arm arm64 mips64le mips64p32le mipsle ppc64le riscv64 + +package main + +import ( + "bytes" + _ "embed" + "fmt" + "io" + + "github.com/cilium/ebpf" +) + +// loadBpf returns the embedded CollectionSpec for bpf. +func loadBpf() (*ebpf.CollectionSpec, error) { + reader := bytes.NewReader(_BpfBytes) + spec, err := ebpf.LoadCollectionSpecFromReader(reader) + if err != nil { + return nil, fmt.Errorf("can't load bpf: %w", err) + } + + return spec, err +} + +// loadBpfObjects loads bpf and converts it into a struct. +// +// The following types are suitable as obj argument: +// +// *bpfObjects +// *bpfPrograms +// *bpfMaps +// +// See ebpf.CollectionSpec.LoadAndAssign documentation for details. +func loadBpfObjects(obj interface{}, opts *ebpf.CollectionOptions) error { + spec, err := loadBpf() + if err != nil { + return err + } + + return spec.LoadAndAssign(obj, opts) +} + +// bpfSpecs contains maps and programs before they are loaded into the kernel. +// +// It can be passed ebpf.CollectionSpec.Assign. +type bpfSpecs struct { + bpfProgramSpecs + bpfMapSpecs +} + +// bpfSpecs contains programs before they are loaded into the kernel. +// +// It can be passed ebpf.CollectionSpec.Assign. +type bpfProgramSpecs struct { + XdpProgFunc *ebpf.ProgramSpec `ebpf:"xdp_prog_func"` +} + +// bpfMapSpecs contains maps before they are loaded into the kernel. +// +// It can be passed ebpf.CollectionSpec.Assign. +type bpfMapSpecs struct { + XdpStatsMap *ebpf.MapSpec `ebpf:"xdp_stats_map"` +} + +// bpfObjects contains all objects after they have been loaded into the kernel. +// +// It can be passed to loadBpfObjects or ebpf.CollectionSpec.LoadAndAssign. +type bpfObjects struct { + bpfPrograms + bpfMaps +} + +func (o *bpfObjects) Close() error { + return _BpfClose( + &o.bpfPrograms, + &o.bpfMaps, + ) +} + +// bpfMaps contains all maps after they have been loaded into the kernel. +// +// It can be passed to loadBpfObjects or ebpf.CollectionSpec.LoadAndAssign. +type bpfMaps struct { + XdpStatsMap *ebpf.Map `ebpf:"xdp_stats_map"` +} + +func (m *bpfMaps) Close() error { + return _BpfClose( + m.XdpStatsMap, + ) +} + +// bpfPrograms contains all programs after they have been loaded into the kernel. +// +// It can be passed to loadBpfObjects or ebpf.CollectionSpec.LoadAndAssign. +type bpfPrograms struct { + XdpProgFunc *ebpf.Program `ebpf:"xdp_prog_func"` +} + +func (p *bpfPrograms) Close() error { + return _BpfClose( + p.XdpProgFunc, + ) +} + +func _BpfClose(closers ...io.Closer) error { + for _, closer := range closers { + if err := closer.Close(); err != nil { + return err + } + } + return nil +} + +// Do not access this directly. +//go:embed bpf_bpfel.o +var _BpfBytes []byte \ No newline at end of file diff --git a/ebpf/ebpf-kernel/main.go b/ebpf/ebpf-kernel/main.go new file mode 100644 index 0000000..6a65fde --- /dev/null +++ b/ebpf/ebpf-kernel/main.go @@ -0,0 +1,144 @@ +//go:build linux +// +build linux + +package main + +import ( + "bytes" + "encoding/binary" + "errors" + "log" + "net" + "os" + "os/signal" + "syscall" + + "golang.org/x/sys/unix" + + "github.com/cilium/ebpf" + "github.com/cilium/ebpf/link" + "github.com/cilium/ebpf/ringbuf" + "github.com/cilium/ebpf/rlimit" +) + +// $BPF_CLANG and $BPF_CFLAGS are set by the Makefile. +//go:generate go run github.com/cilium/ebpf/cmd/bpf2go -cc $BPF_CLANG -cflags $BPF_CFLAGS -type event bpf fentry.c -- -I../headers + +func main() { + if len(os.Args) < 2 { + log.Fatalf("Please specify a network interface") + } + + ifaceName := os.Args[1] + iface, err := net.InterfaceByName(ifaceName) + if err != nil { + log.Fatalf("lookup network iface %q: %s", ifaceName, err) + } + + objs := bpfObjects{} + if err := loadBpfObjects(&objs, nil); err != nil { + log.Fatalf("loading objects: %s", err) + } + defer objs.Close() + + l, err := link.AttachXDP(link.XDPOptions{ + Program: objs.XdpProgFunc, + Interface: iface.Index, + }) + if err != nil { + log.Fatalf("could not attach XDP program: %s", err) + } + defer l.Close() + + log.Printf("Attached XDP program to iface %q (index %d)", iface.Name, iface.Index) + + ticker := time.NewTicker(1 * time.Second) + defer ticker.Stop() + for range ticker.C { + s, err := formatMapContents(objs.XdpStatsMap) + if err != nil { + log.Printf("Error reading map: %s", err) + continue + } + log.Printf("Map contents:\n%s", s) + } +} + +func intToIP(ipNum uint32) net.IP { + ip := make(net.IP, 4) + binary.BigEndian.PutUint32(ip, ipNum) + return ip +} + +func saveDataToMap(desPort, srcAddr, port string) { + // Allow the current process to lock memory for eBPF resources. + if err := rlimit.RemoveMemlock(); err != nil { + log.Fatal(err) + } + + outerMapSpec := ebpf.MapSpec{ + Name: "outer_map", + Type: ebpf.ArrayOfMaps, + KeySize: 10000, + ValueSize: 10000, + MaxEntries: 10000, + Contents: make([]ebpf.MapKV, 5), + InnerMap: &ebpf.MapSpec{ + Name: "inner_map", + Type: ebpf.Array, + KeySize: 100, + ValueSize: 100, + Flags: unix.BPF_F_INNER_MAP, + MaxEntries: 1, + }, + } + + innerMapSpec := outerMapSpec.InnerMap.Copy() + innerMapSpec.MaxEntries = 1000 + innerMapSpec.Contents = make([]ebpf.MapKV, innerMapSpec.MaxEntries) + innerMapSpec = append(innerMapSpec, ebpf.MapKV{Key: desPort, Value: srcAddr+port}) + innerMap, err := ebpf.NewMap(innerMapSpec) + if err != nil { + log.Fatalf("inner_map: %v", err) + } + defer innerMap.Close() + outerMapSpec.Contents = append(outerMapSpec.Contents, ebpf.MapKV{Key: i, Value: innerMap}) +} + +func readMap(outerMapSpec ebpf.MapSpec) { + outerMap, err := ebpf.NewMap(&outerMapSpec) + if err != nil { + log.Fatalf("outer_map: %v", err) + } + defer outerMap.Close() + + mapIter := outerMap.Iterate() + var outerMapKey uint32 + var innerMapID ebpf.MapID + for mapIter.Next(&outerMapKey, &innerMapID) { + innerMap, err := ebpf.NewMapFromID(innerMapID) + if err != nil { + log.Fatal(err) + } + innerMapInfo, err := innerMap.Info() + if err != nil { + log.Fatal(err) + } + log.Printf("outerMapKey %d, innerMap.Info: %+v", outerMapKey, innerMapInfo.) + } +} + +func formatMapContents(m *ebpf.Map) (string, error) { + var ( + sb strings.Builder + key []byte + val uint32 + ) + iter := m.Iterate() + for iter.Next(&key, &val) { + sourceIP := net.IP(key) // IPv4 source address in network byte order. + packetCount := val + sb.WriteString(fmt.Sprintf("\t%s => %d\n", sourceIP, packetCount)) + } + return sb.String(), iter.Err() +} \ No newline at end of file diff --git a/ebpf/ebpf-kernel/tcprtt.c b/ebpf/ebpf-kernel/tcprtt.c new file mode 100644 index 0000000..45da9e6 --- /dev/null +++ b/ebpf/ebpf-kernel/tcprtt.c @@ -0,0 +1,93 @@ +// +build ignore + +#include "common.h" + +#include "bpf_endian.h" +#include "bpf_tracing.h" + +#define AF_INET 2 +#define TASK_COMM_LEN 16 + +char __license[] SEC("license") = "Dual MIT/GPL"; + +/** + * struct sock_common reflects the start of the kernel's struct sock_common. + * It only contains the fields up until skc_family that are accessed in the + * program, with padding to match the kernel's declaration. + */ +struct sock_common +{ + union + { + struct + { + __be32 skc_daddr; + __be32 skc_rcv_saddr; + }; + }; + union + { + // Padding out union skc_hash. + __u32 _; + }; + union + { + struct + { + __be16 skc_dport; + __u16 skc_num; + }; + }; + short unsigned int skc_family; +}; + +/** + * struct sock reflects the start of the kernel's struct sock. + */ +struct sock +{ + struct sock_common __sk_common; +}; + +struct +{ + __uint(type, BPF_MAP_TYPE_RINGBUF); + __uint(max_entries, 1 << 24); +} events SEC(".maps"); + +struct event +{ + u8 comm[16]; + __u16 sport; + __be16 dport; + __be32 saddr; + __be32 daddr; +}; +struct event *unused __attribute__((unused)); + +SEC("fentry/tcp_connect") +int BPF_PROG(tcp_connect, struct sock *sk) +{ + if (sk->__sk_common.skc_family != AF_INET) + { + return 0; + } + + struct event *tcp_info; + tcp_info = bpf_ringbuf_reserve(&events, sizeof(struct event), 0); + if (!tcp_info) + { + return 0; + } + + tcp_info->saddr = sk->__sk_common.skc_rcv_saddr; + tcp_info->daddr = sk->__sk_common.skc_daddr; + tcp_info->dport = sk->__sk_common.skc_dport; + tcp_info->sport = bpf_htons(sk->__sk_common.skc_num); + + bpf_get_current_comm(&tcp_info->comm, TASK_COMM_LEN); + + bpf_ringbuf_submit(tcp_info, 0); + + return 0; +} diff --git a/ebpf/ebpf-kernel/xdp.c b/ebpf/ebpf-kernel/xdp.c new file mode 100644 index 0000000..ae1feec --- /dev/null +++ b/ebpf/ebpf-kernel/xdp.c @@ -0,0 +1,65 @@ +#include "bpf_endian.h" +#include "common.h" + +char __license[] SEC("license") = "Dual MIT/GPL"; + +#define MAX_MAP_ENTRIES 16 + +/* Define an LRU hash map for storing packet count by source IPv4 address */ +struct +{ + __uint(type, BPF_MAP_TYPE_LRU_HASH); + __uint(max_entries, MAX_MAP_ENTRIES); + __type(key, __u32); // source IPv4 address + __type(value, __u32); // packet count +} xdp_stats_map SEC(".maps"); + +static __always_inline int parse_ip_src_addr(struct xdp_md *ctx, __u32 *ip_src_addr) +{ + void *data_end = (void *)(long)ctx->data_end; + void *data = (void *)(long)ctx->data; + + struct ethhdr *eth = data; + if ((void *)(eth + 1) > data_end) + { + return 0; + } + + if (eth->h_proto != bpf_htons(ETH_P_IP)) + { + return 0; + } + + struct iphdr *ip = (void *)(eth + 1); + if ((void *)(ip + 1) > data_end) + { + return 0; + } + + *ip_src_addr = (__u32)(ip->saddr); + return 1; +} + +SEC("xdp") +int xdp_prog_func(struct xdp_md *ctx) +{ + __u32 ip; + if (!parse_ip_src_addr(ctx, &ip)) + { + goto done; + } + + __u32 *pkt_count = bpf_map_lookup_elem(&xdp_stats_map, &ip); + if (!pkt_count) + { + __u32 init_pkt_count = 1; + bpf_map_update_elem(&xdp_stats_map, &ip, &init_pkt_count, BPF_ANY); + } + else + { + __sync_fetch_and_add(pkt_count, 1); + } + +done: + return XDP_PASS; +} diff --git a/ebpf/ebpf-user/main.go b/ebpf/ebpf-user/main.go new file mode 100644 index 0000000..ba2bf8f --- /dev/null +++ b/ebpf/ebpf-user/main.go @@ -0,0 +1,32 @@ +package main + +import ( + "log" + + "github.com/cilium/ebpf" +) + +func main() { + // outerMap, err := ebpf.NewMap(&outerMapSpec) + // if err != nil { + // log.Fatalf("outer_map: %v", err) + // } + // defer outerMap.Close() + + mapIter := outerMap.Iterate() + var outerMapKey uint32 + var innerMapID ebpf.MapID + for mapIter.Next(&outerMapKey, &innerMapID) { + innerMap, err := ebpf.NewMapFromID(innerMapID) + if err != nil { + log.Fatal(err) + } + + innerMapInfo, err := innerMap.Info() + if err != nil { + log.Fatal(err) + } + + log.Printf("outerMapKey %d, innerMap.Info: %+v", outerMapKey, innerMapInfo) + } +} From bf004c23754ec578a775871b58b87080bc721a9f Mon Sep 17 00:00:00 2001 From: halfrost Date: Mon, 31 Oct 2022 09:40:48 -0700 Subject: [PATCH 2/2] Add README and makefile --- README.md | 107 ++++++++++++++++++++++++++- ebpf/{ebpf-kernel => }/bpf_bpfeb.go | 21 ++++-- ebpf/{ebpf-kernel => }/bpf_bpfel.go | 23 ++++-- ebpf/ebpf-kernel/xdp.c | 65 ---------------- ebpf/ebpf-user/main.go | 32 -------- ebpf/{ebpf-kernel => }/main.go | 82 ++++++++++++++------ ebpf/{ebpf-kernel/tcprtt.c => tcp.c} | 0 make | 41 ++++++++++ 8 files changed, 236 insertions(+), 135 deletions(-) rename ebpf/{ebpf-kernel => }/bpf_bpfeb.go (89%) rename ebpf/{ebpf-kernel => }/bpf_bpfel.go (89%) delete mode 100644 ebpf/ebpf-kernel/xdp.c delete mode 100644 ebpf/ebpf-user/main.go rename ebpf/{ebpf-kernel => }/main.go (64%) rename ebpf/{ebpf-kernel/tcprtt.c => tcp.c} (100%) create mode 100644 make diff --git a/README.md b/README.md index 15538f5..812b263 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,107 @@ # sentinel-go-ebpf-plugin -Sentinel eBPF plugin implemented by golang language + +This is an exploratory project. The purpose is to use the function of ebpf to implement some features such as sentinel traffic limit. The project is still being explored. + +# Requirements +Linux >= 4.9. + +# Example + +use `go generate` to compile the code. + +```go +$ make -C ../ +``` + +Then + +```go +$ go run -exec sudo ./ebpf +``` + +Output as below: + +```go +2022/10/31 16:27:45 Comm Src addr Port -> Dest addr Port +2022/10/31 16:27:54 google_guest_ag 10.182.0.2 55146 -> 74.125.137.95 443 +2022/10/31 16:28:11 google_osconfig 10.182.0.2 35330 -> 74.125.137.95 443 +2022/10/31 16:31:54 google_guest_ag 10.182.0.2 38566 -> 142.251.2.95 443 +2022/10/31 16:32:11 google_osconfig 10.182.0.2 59934 -> 142.251.2.95 443 +2022/10/31 16:35:54 google_guest_ag 10.182.0.2 54706 -> 142.251.2.95 443 +2022/10/31 16:36:11 google_osconfig 10.182.0.2 39752 -> 142.251.2.95 443 +2022/10/31 16:39:54 google_guest_ag 10.182.0.2 42938 -> 142.250.141.95 443 +2022/10/31 16:40:11 google_osconfig 10.182.0.2 52648 -> 142.250.141.95 443 +``` + + +# Waht we can get + +```go +/* user accessible mirror of in-kernel sk_buff. + * new fields can only be added to the end of this structure + */ +struct __sk_buff { + __u32 len; + __u32 pkt_type; + __u32 mark; + __u32 queue_mapping; + __u32 protocol; + __u32 vlan_present; + __u32 vlan_tci; + __u32 vlan_proto; + __u32 priority; + __u32 ingress_ifindex; + __u32 ifindex; + __u32 tc_index; + __u32 cb[5]; + __u32 hash; + __u32 tc_classid; + __u32 data; + __u32 data_end; + __u32 napi_id; + + /* Accessed by BPF_PROG_TYPE_sk_skb types from here to ... */ + __u32 family; + __u32 remote_ip4; /* Stored in network byte order */ + __u32 local_ip4; /* Stored in network byte order */ + __u32 remote_ip6[4]; /* Stored in network byte order */ + __u32 local_ip6[4]; /* Stored in network byte order */ + __u32 remote_port; /* Stored in network byte order */ + __u32 local_port; /* stored in host byte order */ + /* ... here. */ + + __u32 data_meta; + __bpf_md_ptr(struct bpf_flow_keys *, flow_keys); + __u64 tstamp; + __u32 wire_len; + __u32 gso_segs; + __bpf_md_ptr(struct bpf_sock *, sk); + __u32 gso_size; +}; +``` + +Some tcp header information: + +```go + __u32 remote_ip4; /* Stored in network byte order */ + __u32 local_ip4; /* Stored in network byte order */ + __u32 remote_ip6[4]; /* Stored in network byte order */ + __u32 local_ip6[4]; /* Stored in network byte order */ + __u32 remote_port; /* Stored in network byte order */ + __u32 local_port; /* stored in host byte order */ +``` + +data\_meta store the meta data. +data is the pointer to the begin of data. +data_end is the pointer to the end of data. + +Using the data pointer, the starting point of each tcp data can be found. But if you want to get the entire http package, you need to get multiple tcp packages, which is more difficult. + +# To continue research +- use kprobe bpf to hook system call of application process +- use uprobe bpf to hook entire http package + + +# License +MIT + diff --git a/ebpf/ebpf-kernel/bpf_bpfeb.go b/ebpf/bpf_bpfeb.go similarity index 89% rename from ebpf/ebpf-kernel/bpf_bpfeb.go rename to ebpf/bpf_bpfeb.go index c73697a..19984d7 100644 --- a/ebpf/ebpf-kernel/bpf_bpfeb.go +++ b/ebpf/bpf_bpfeb.go @@ -13,6 +13,14 @@ import ( "github.com/cilium/ebpf" ) +type bpfEvent struct { + Comm [16]uint8 + Sport uint16 + Dport uint16 + Saddr uint32 + Daddr uint32 +} + // loadBpf returns the embedded CollectionSpec for bpf. func loadBpf() (*ebpf.CollectionSpec, error) { reader := bytes.NewReader(_BpfBytes) @@ -54,14 +62,14 @@ type bpfSpecs struct { // // It can be passed ebpf.CollectionSpec.Assign. type bpfProgramSpecs struct { - XdpProgFunc *ebpf.ProgramSpec `ebpf:"xdp_prog_func"` + TcpConnect *ebpf.ProgramSpec `ebpf:"tcp_connect"` } // bpfMapSpecs contains maps before they are loaded into the kernel. // // It can be passed ebpf.CollectionSpec.Assign. type bpfMapSpecs struct { - XdpStatsMap *ebpf.MapSpec `ebpf:"xdp_stats_map"` + Events *ebpf.MapSpec `ebpf:"events"` } // bpfObjects contains all objects after they have been loaded into the kernel. @@ -83,12 +91,12 @@ func (o *bpfObjects) Close() error { // // It can be passed to loadBpfObjects or ebpf.CollectionSpec.LoadAndAssign. type bpfMaps struct { - XdpStatsMap *ebpf.Map `ebpf:"xdp_stats_map"` + Events *ebpf.Map `ebpf:"events"` } func (m *bpfMaps) Close() error { return _BpfClose( - m.XdpStatsMap, + m.Events, ) } @@ -96,12 +104,12 @@ func (m *bpfMaps) Close() error { // // It can be passed to loadBpfObjects or ebpf.CollectionSpec.LoadAndAssign. type bpfPrograms struct { - XdpProgFunc *ebpf.Program `ebpf:"xdp_prog_func"` + TcpConnect *ebpf.Program `ebpf:"tcp_connect"` } func (p *bpfPrograms) Close() error { return _BpfClose( - p.XdpProgFunc, + p.TcpConnect, ) } @@ -115,5 +123,6 @@ func _BpfClose(closers ...io.Closer) error { } // Do not access this directly. +// //go:embed bpf_bpfeb.o var _BpfBytes []byte \ No newline at end of file diff --git a/ebpf/ebpf-kernel/bpf_bpfel.go b/ebpf/bpf_bpfel.go similarity index 89% rename from ebpf/ebpf-kernel/bpf_bpfel.go rename to ebpf/bpf_bpfel.go index f391329..3777122 100644 --- a/ebpf/ebpf-kernel/bpf_bpfel.go +++ b/ebpf/bpf_bpfel.go @@ -1,5 +1,3 @@ - - // Code generated by bpf2go; DO NOT EDIT. //go:build 386 || amd64 || amd64p32 || arm || arm64 || mips64le || mips64p32le || mipsle || ppc64le || riscv64 // +build 386 amd64 amd64p32 arm arm64 mips64le mips64p32le mipsle ppc64le riscv64 @@ -15,6 +13,14 @@ import ( "github.com/cilium/ebpf" ) +type bpfEvent struct { + Comm [16]uint8 + Sport uint16 + Dport uint16 + Saddr uint32 + Daddr uint32 +} + // loadBpf returns the embedded CollectionSpec for bpf. func loadBpf() (*ebpf.CollectionSpec, error) { reader := bytes.NewReader(_BpfBytes) @@ -56,14 +62,14 @@ type bpfSpecs struct { // // It can be passed ebpf.CollectionSpec.Assign. type bpfProgramSpecs struct { - XdpProgFunc *ebpf.ProgramSpec `ebpf:"xdp_prog_func"` + TcpConnect *ebpf.ProgramSpec `ebpf:"tcp_connect"` } // bpfMapSpecs contains maps before they are loaded into the kernel. // // It can be passed ebpf.CollectionSpec.Assign. type bpfMapSpecs struct { - XdpStatsMap *ebpf.MapSpec `ebpf:"xdp_stats_map"` + Events *ebpf.MapSpec `ebpf:"events"` } // bpfObjects contains all objects after they have been loaded into the kernel. @@ -85,12 +91,12 @@ func (o *bpfObjects) Close() error { // // It can be passed to loadBpfObjects or ebpf.CollectionSpec.LoadAndAssign. type bpfMaps struct { - XdpStatsMap *ebpf.Map `ebpf:"xdp_stats_map"` + Events *ebpf.Map `ebpf:"events"` } func (m *bpfMaps) Close() error { return _BpfClose( - m.XdpStatsMap, + m.Events, ) } @@ -98,12 +104,12 @@ func (m *bpfMaps) Close() error { // // It can be passed to loadBpfObjects or ebpf.CollectionSpec.LoadAndAssign. type bpfPrograms struct { - XdpProgFunc *ebpf.Program `ebpf:"xdp_prog_func"` + TcpConnect *ebpf.Program `ebpf:"tcp_connect"` } func (p *bpfPrograms) Close() error { return _BpfClose( - p.XdpProgFunc, + p.TcpConnect, ) } @@ -117,5 +123,6 @@ func _BpfClose(closers ...io.Closer) error { } // Do not access this directly. +// //go:embed bpf_bpfel.o var _BpfBytes []byte \ No newline at end of file diff --git a/ebpf/ebpf-kernel/xdp.c b/ebpf/ebpf-kernel/xdp.c deleted file mode 100644 index ae1feec..0000000 --- a/ebpf/ebpf-kernel/xdp.c +++ /dev/null @@ -1,65 +0,0 @@ -#include "bpf_endian.h" -#include "common.h" - -char __license[] SEC("license") = "Dual MIT/GPL"; - -#define MAX_MAP_ENTRIES 16 - -/* Define an LRU hash map for storing packet count by source IPv4 address */ -struct -{ - __uint(type, BPF_MAP_TYPE_LRU_HASH); - __uint(max_entries, MAX_MAP_ENTRIES); - __type(key, __u32); // source IPv4 address - __type(value, __u32); // packet count -} xdp_stats_map SEC(".maps"); - -static __always_inline int parse_ip_src_addr(struct xdp_md *ctx, __u32 *ip_src_addr) -{ - void *data_end = (void *)(long)ctx->data_end; - void *data = (void *)(long)ctx->data; - - struct ethhdr *eth = data; - if ((void *)(eth + 1) > data_end) - { - return 0; - } - - if (eth->h_proto != bpf_htons(ETH_P_IP)) - { - return 0; - } - - struct iphdr *ip = (void *)(eth + 1); - if ((void *)(ip + 1) > data_end) - { - return 0; - } - - *ip_src_addr = (__u32)(ip->saddr); - return 1; -} - -SEC("xdp") -int xdp_prog_func(struct xdp_md *ctx) -{ - __u32 ip; - if (!parse_ip_src_addr(ctx, &ip)) - { - goto done; - } - - __u32 *pkt_count = bpf_map_lookup_elem(&xdp_stats_map, &ip); - if (!pkt_count) - { - __u32 init_pkt_count = 1; - bpf_map_update_elem(&xdp_stats_map, &ip, &init_pkt_count, BPF_ANY); - } - else - { - __sync_fetch_and_add(pkt_count, 1); - } - -done: - return XDP_PASS; -} diff --git a/ebpf/ebpf-user/main.go b/ebpf/ebpf-user/main.go deleted file mode 100644 index ba2bf8f..0000000 --- a/ebpf/ebpf-user/main.go +++ /dev/null @@ -1,32 +0,0 @@ -package main - -import ( - "log" - - "github.com/cilium/ebpf" -) - -func main() { - // outerMap, err := ebpf.NewMap(&outerMapSpec) - // if err != nil { - // log.Fatalf("outer_map: %v", err) - // } - // defer outerMap.Close() - - mapIter := outerMap.Iterate() - var outerMapKey uint32 - var innerMapID ebpf.MapID - for mapIter.Next(&outerMapKey, &innerMapID) { - innerMap, err := ebpf.NewMapFromID(innerMapID) - if err != nil { - log.Fatal(err) - } - - innerMapInfo, err := innerMap.Info() - if err != nil { - log.Fatal(err) - } - - log.Printf("outerMapKey %d, innerMap.Info: %+v", outerMapKey, innerMapInfo) - } -} diff --git a/ebpf/ebpf-kernel/main.go b/ebpf/main.go similarity index 64% rename from ebpf/ebpf-kernel/main.go rename to ebpf/main.go index 6a65fde..696df23 100644 --- a/ebpf/ebpf-kernel/main.go +++ b/ebpf/main.go @@ -7,15 +7,14 @@ import ( "bytes" "encoding/binary" "errors" + "fmt" "log" "net" "os" "os/signal" + "strings" "syscall" - "golang.org/x/sys/unix" - - "github.com/cilium/ebpf" "github.com/cilium/ebpf/link" "github.com/cilium/ebpf/ringbuf" "github.com/cilium/ebpf/rlimit" @@ -25,45 +24,82 @@ import ( //go:generate go run github.com/cilium/ebpf/cmd/bpf2go -cc $BPF_CLANG -cflags $BPF_CFLAGS -type event bpf fentry.c -- -I../headers func main() { - if len(os.Args) < 2 { - log.Fatalf("Please specify a network interface") - } + stopper := make(chan os.Signal, 1) + signal.Notify(stopper, os.Interrupt, syscall.SIGTERM) - ifaceName := os.Args[1] - iface, err := net.InterfaceByName(ifaceName) - if err != nil { - log.Fatalf("lookup network iface %q: %s", ifaceName, err) + // Allow the current process to lock memory for eBPF resources. + if err := rlimit.RemoveMemlock(); err != nil { + log.Fatal(err) } + // Load pre-compiled programs and maps into the kernel. objs := bpfObjects{} if err := loadBpfObjects(&objs, nil); err != nil { - log.Fatalf("loading objects: %s", err) + log.Fatalf("loading objects: %v", err) } defer objs.Close() - l, err := link.AttachXDP(link.XDPOptions{ - Program: objs.XdpProgFunc, - Interface: iface.Index, + link, err := link.AttachTracing(link.TracingOptions{ + Program: objs.bpfPrograms.TcpConnect, }) if err != nil { - log.Fatalf("could not attach XDP program: %s", err) + log.Fatal(err) } - defer l.Close() + defer link.Close() + + rd, err := ringbuf.NewReader(objs.bpfMaps.Events) + if err != nil { + log.Fatalf("opening ringbuf reader: %s", err) + } + defer rd.Close() + + go func() { + <-stopper - log.Printf("Attached XDP program to iface %q (index %d)", iface.Name, iface.Index) + if err := rd.Close(); err != nil { + log.Fatalf("closing ringbuf reader: %s", err) + } + }() + + log.Printf("%-16s %-15s %-6s -> %-15s %-6s", + "Comm", + "Src addr", + "Port", + "Dest addr", + "Port", + ) - ticker := time.NewTicker(1 * time.Second) - defer ticker.Stop() - for range ticker.C { - s, err := formatMapContents(objs.XdpStatsMap) + // bpfEvent is generated by bpf2go. + var event bpfEvent + for { + record, err := rd.Read() if err != nil { - log.Printf("Error reading map: %s", err) + if errors.Is(err, ringbuf.ErrClosed) { + log.Println("received signal, exiting..") + return + } + log.Printf("reading from reader: %s", err) continue } - log.Printf("Map contents:\n%s", s) + + // Parse the ringbuf event entry into a bpfEvent structure. + if err := binary.Read(bytes.NewBuffer(record.RawSample), binary.BigEndian, &event); err != nil { + log.Printf("parsing ringbuf event: %s", err) + continue + } + + log.Printf("%-16s %-15s %-6d -> %-15s %-6d", + event.Comm, + intToIP(event.Saddr), + event.Sport, + intToIP(event.Daddr), + event.Dport, + ) + saveDataToMap() } } +// intToIP converts IPv4 number to net.IP func intToIP(ipNum uint32) net.IP { ip := make(net.IP, 4) binary.BigEndian.PutUint32(ip, ipNum) diff --git a/ebpf/ebpf-kernel/tcprtt.c b/ebpf/tcp.c similarity index 100% rename from ebpf/ebpf-kernel/tcprtt.c rename to ebpf/tcp.c diff --git a/make b/make new file mode 100644 index 0000000..da7f062 --- /dev/null +++ b/make @@ -0,0 +1,41 @@ +# The development version of clang is distributed as the 'clang' binary, +# while stable/released versions have a version number attached. +# Pin the default clang to a stable version. +CLANG ?= clang-14 +STRIP ?= llvm-strip-14 +OBJCOPY ?= llvm-objcopy-14 +CFLAGS := -O2 -g -Wall -Werror $(CFLAGS) + +# Obtain an absolute path to the directory of the Makefile. +# Assume the Makefile is in the root of the repository. +REPODIR := $(shell dirname $(realpath $(firstword $(MAKEFILE_LIST)))) +UIDGID := $(shell stat -c '%u:%g' ${REPODIR}) + +# Prefer podman if installed, otherwise use docker. +# Note: Setting the var at runtime will always override. +CONTAINER_ENGINE ?= $(if $(shell command -v podman), podman, docker) +CONTAINER_RUN_ARGS ?= $(if $(filter ${CONTAINER_ENGINE}, podman), --log-driver=none, --user "${UIDGID}") + +IMAGE := $(shell cat ${REPODIR}/testdata/docker/IMAGE) +VERSION := $(shell cat ${REPODIR}/testdata/docker/VERSION) + +clean: + -$(RM) testdata/*.elf + -$(RM) btf/testdata/*.elf + +format: + find . -type f -name "*.c" | xargs clang-format -i + +# $BPF_CLANG is used in go:generate invocations. +generate: export BPF_CLANG := $(CLANG) +generate: export BPF_CFLAGS := $(CFLAGS) +generate: + go generate ./... + +%-el.elf: %.c + $(CLANG) $(CFLAGS) -target bpfel -c $< -o $@ + $(STRIP) -g $@ + +%-eb.elf : %.c + $(CLANG) $(CFLAGS) -target bpfeb -c $< -o $@ + $(STRIP) -g $@ \ No newline at end of file