diff --git a/bpf/bpf.go b/bpf/bpf.go index c9df323e..ff66780b 100644 --- a/bpf/bpf.go +++ b/bpf/bpf.go @@ -33,10 +33,12 @@ type BPF struct { opts Options closeFuncs []func() - skipAttachCgroup bool - skipTcx bool - isLegacyKernel bool - report *types.CountReport + skipAttachCgroup bool + skipTcx bool + isLegacyKernel bool + supportRingBuf bool + useRingBufSubmitSkb bool + report *types.CountReport } type Options struct { @@ -87,12 +89,14 @@ func NewBPF() (*BPF, error) { } bf := &BPF{ - spec: spec, - objs: &BpfObjects{}, - report: &types.CountReport{}, - isLegacyKernel: legacyKernel, - skipAttachCgroup: skipAttachCgroup, - skipTcx: !supportTcx(), + spec: spec, + objs: &BpfObjects{}, + report: &types.CountReport{}, + isLegacyKernel: legacyKernel, + skipAttachCgroup: skipAttachCgroup, + skipTcx: !supportTcx(), + supportRingBuf: !legacyKernel && supportRingBuf(), + useRingBufSubmitSkb: canUseRingBufSubmitSkb(), } return bf, nil @@ -113,6 +117,9 @@ func (b *BPF) Load(opts Options) error { if len(opts.ifindexes) > 0 { config.FilterIfindexEnable = 1 } + if b.useRingBufSubmitSkb { + config.UseRingbufSubmitSkb = 1 + } if opts.backend != types.NetHookBackendCgroupSkb { b.disableCgroupSkb() } @@ -142,6 +149,7 @@ load: IgnoreUnknownProgram: true, IgnoreNotSupportedProgram: true, IgnoreUnknownVariable: true, + IgnoreInvalidMap: true, }) if err != nil { log.Infof("load and assign failed: %+v", err) @@ -323,6 +331,7 @@ func (b *BPF) attachTcxHooks(ifindex int, egress, ingress bool) ([]func(), error } if egress { + log.Infof("attach tcx/egress hooks to ifindex %d", ifindex) lk, err := link.AttachTCX(link.TCXOptions{ Interface: ifindex, Program: b.objs.TcxEgress, @@ -337,6 +346,7 @@ func (b *BPF) attachTcxHooks(ifindex int, egress, ingress bool) ([]func(), error } if ingress { + log.Infof("attach tcx/ingress hooks to ifindex %d", ifindex) lk, err := link.AttachTCX(link.TCXOptions{ Interface: ifindex, Program: b.objs.TcxIngress, diff --git a/bpf/bpf_arm64_bpfel.go b/bpf/bpf_arm64_bpfel.go index 4fcf88b0..7cc44226 100644 --- a/bpf/bpf_arm64_bpfel.go +++ b/bpf/bpf_arm64_bpfel.go @@ -42,6 +42,8 @@ type BpfGconfigT struct { FilterComm [16]int8 FilterCommEnable uint8 FilterIfindexEnable uint8 + UseRingbufSubmitSkb uint8 + _ [3]byte MaxPayloadSize uint32 } @@ -211,7 +213,10 @@ type BpfMapSpecs struct { EnterMountBufs *ebpf.MapSpec `ebpf:"enter_mount_bufs"` ExecEventStack *ebpf.MapSpec `ebpf:"exec_event_stack"` ExecEvents *ebpf.MapSpec `ebpf:"exec_events"` + ExecEventsRingbuf *ebpf.MapSpec `ebpf:"exec_events_ringbuf"` + ExitEventTmp *ebpf.MapSpec `ebpf:"exit_event_tmp"` ExitEvents *ebpf.MapSpec `ebpf:"exit_events"` + ExitEventsRingbuf *ebpf.MapSpec `ebpf:"exit_events_ringbuf"` FilterByKernelCount *ebpf.MapSpec `ebpf:"filter_by_kernel_count"` FilterIfindexMap *ebpf.MapSpec `ebpf:"filter_ifindex_map"` FilterMntnsMap *ebpf.MapSpec `ebpf:"filter_mntns_map"` @@ -221,7 +226,9 @@ type BpfMapSpecs struct { FilterUidMap *ebpf.MapSpec `ebpf:"filter_uid_map"` FlowPidMap *ebpf.MapSpec `ebpf:"flow_pid_map"` GoKeylogBufStorage *ebpf.MapSpec `ebpf:"go_keylog_buf_storage"` + GoKeylogEventTmp *ebpf.MapSpec `ebpf:"go_keylog_event_tmp"` GoKeylogEvents *ebpf.MapSpec `ebpf:"go_keylog_events"` + GoKeylogEventsRingbuf *ebpf.MapSpec `ebpf:"go_keylog_events_ringbuf"` MountEventStack *ebpf.MapSpec `ebpf:"mount_event_stack"` MountEvents *ebpf.MapSpec `ebpf:"mount_events"` NatFlowMap *ebpf.MapSpec `ebpf:"nat_flow_map"` @@ -230,6 +237,7 @@ type BpfMapSpecs struct { NewNetdeviceEvents *ebpf.MapSpec `ebpf:"new_netdevice_events"` PacketEventStack *ebpf.MapSpec `ebpf:"packet_event_stack"` PacketEvents *ebpf.MapSpec `ebpf:"packet_events"` + PacketEventsRingbuf *ebpf.MapSpec `ebpf:"packet_events_ringbuf"` SockCookiePidMap *ebpf.MapSpec `ebpf:"sock_cookie_pid_map"` TidNetdeviceMap *ebpf.MapSpec `ebpf:"tid_netdevice_map"` } @@ -274,7 +282,10 @@ type BpfMaps struct { EnterMountBufs *ebpf.Map `ebpf:"enter_mount_bufs"` ExecEventStack *ebpf.Map `ebpf:"exec_event_stack"` ExecEvents *ebpf.Map `ebpf:"exec_events"` + ExecEventsRingbuf *ebpf.Map `ebpf:"exec_events_ringbuf"` + ExitEventTmp *ebpf.Map `ebpf:"exit_event_tmp"` ExitEvents *ebpf.Map `ebpf:"exit_events"` + ExitEventsRingbuf *ebpf.Map `ebpf:"exit_events_ringbuf"` FilterByKernelCount *ebpf.Map `ebpf:"filter_by_kernel_count"` FilterIfindexMap *ebpf.Map `ebpf:"filter_ifindex_map"` FilterMntnsMap *ebpf.Map `ebpf:"filter_mntns_map"` @@ -284,7 +295,9 @@ type BpfMaps struct { FilterUidMap *ebpf.Map `ebpf:"filter_uid_map"` FlowPidMap *ebpf.Map `ebpf:"flow_pid_map"` GoKeylogBufStorage *ebpf.Map `ebpf:"go_keylog_buf_storage"` + GoKeylogEventTmp *ebpf.Map `ebpf:"go_keylog_event_tmp"` GoKeylogEvents *ebpf.Map `ebpf:"go_keylog_events"` + GoKeylogEventsRingbuf *ebpf.Map `ebpf:"go_keylog_events_ringbuf"` MountEventStack *ebpf.Map `ebpf:"mount_event_stack"` MountEvents *ebpf.Map `ebpf:"mount_events"` NatFlowMap *ebpf.Map `ebpf:"nat_flow_map"` @@ -293,6 +306,7 @@ type BpfMaps struct { NewNetdeviceEvents *ebpf.Map `ebpf:"new_netdevice_events"` PacketEventStack *ebpf.Map `ebpf:"packet_event_stack"` PacketEvents *ebpf.Map `ebpf:"packet_events"` + PacketEventsRingbuf *ebpf.Map `ebpf:"packet_events_ringbuf"` SockCookiePidMap *ebpf.Map `ebpf:"sock_cookie_pid_map"` TidNetdeviceMap *ebpf.Map `ebpf:"tid_netdevice_map"` } @@ -303,7 +317,10 @@ func (m *BpfMaps) Close() error { m.EnterMountBufs, m.ExecEventStack, m.ExecEvents, + m.ExecEventsRingbuf, + m.ExitEventTmp, m.ExitEvents, + m.ExitEventsRingbuf, m.FilterByKernelCount, m.FilterIfindexMap, m.FilterMntnsMap, @@ -313,7 +330,9 @@ func (m *BpfMaps) Close() error { m.FilterUidMap, m.FlowPidMap, m.GoKeylogBufStorage, + m.GoKeylogEventTmp, m.GoKeylogEvents, + m.GoKeylogEventsRingbuf, m.MountEventStack, m.MountEvents, m.NatFlowMap, @@ -322,6 +341,7 @@ func (m *BpfMaps) Close() error { m.NewNetdeviceEvents, m.PacketEventStack, m.PacketEvents, + m.PacketEventsRingbuf, m.SockCookiePidMap, m.TidNetdeviceMap, ) diff --git a/bpf/bpf_arm64_bpfel.o b/bpf/bpf_arm64_bpfel.o index a55d9ed7..ca4b24d2 100644 Binary files a/bpf/bpf_arm64_bpfel.o and b/bpf/bpf_arm64_bpfel.o differ diff --git a/bpf/bpf_legacy.go b/bpf/bpf_legacy.go index 82af6d75..02a213dd 100644 --- a/bpf/bpf_legacy.go +++ b/bpf/bpf_legacy.go @@ -54,6 +54,42 @@ func supportTcx() bool { return versionCode >= kernelVersion(6, 6, 0) } +func supportRingBuf() bool { + log.Info("Checking ringbuf support") + if ok := kernelVersionEqOrGreaterThan(5, 8, 0); !ok { + return false + } + if err := features.HaveMapType(ebpf.RingBuf); err != nil { + log.Infof("%+v", err) + return false + } + return true +} + +func canUseRingBufSubmitSkb() bool { + log.Info("Checking ringbuf submit skb support") + if !supportRingBuf() { + return false + } + // 5.8 ~ 6.7 will raise "R3 min value is outside of the allowed memory range" error + if ok := kernelVersionEqOrGreaterThan(6, 8, 0); ok { + return true + } + return false +} + +func kernelVersionEqOrGreaterThan(a, b, c int) bool { + versionCode, err := features.LinuxVersionCode() + if err != nil { + log.Infof("%+v", err) + return false + } + if versionCode >= kernelVersion(a, b, c) { + return true + } + return false +} + func loadBpfWithData(b []byte) (*ebpf.CollectionSpec, error) { reader := bytes.NewReader(b) spec, err := ebpf.LoadCollectionSpecFromReader(reader) diff --git a/bpf/bpf_legacy_arm64_bpfel.go b/bpf/bpf_legacy_arm64_bpfel.go index 53a915cb..cdbc140f 100644 --- a/bpf/bpf_legacy_arm64_bpfel.go +++ b/bpf/bpf_legacy_arm64_bpfel.go @@ -87,7 +87,10 @@ type bpf_legacyMapSpecs struct { EnterMountBufs *ebpf.MapSpec `ebpf:"enter_mount_bufs"` ExecEventStack *ebpf.MapSpec `ebpf:"exec_event_stack"` ExecEvents *ebpf.MapSpec `ebpf:"exec_events"` + ExecEventsRingbuf *ebpf.MapSpec `ebpf:"exec_events_ringbuf"` + ExitEventTmp *ebpf.MapSpec `ebpf:"exit_event_tmp"` ExitEvents *ebpf.MapSpec `ebpf:"exit_events"` + ExitEventsRingbuf *ebpf.MapSpec `ebpf:"exit_events_ringbuf"` FilterByKernelCount *ebpf.MapSpec `ebpf:"filter_by_kernel_count"` FilterIfindexMap *ebpf.MapSpec `ebpf:"filter_ifindex_map"` FilterMntnsMap *ebpf.MapSpec `ebpf:"filter_mntns_map"` @@ -97,7 +100,9 @@ type bpf_legacyMapSpecs struct { FilterUidMap *ebpf.MapSpec `ebpf:"filter_uid_map"` FlowPidMap *ebpf.MapSpec `ebpf:"flow_pid_map"` GoKeylogBufStorage *ebpf.MapSpec `ebpf:"go_keylog_buf_storage"` + GoKeylogEventTmp *ebpf.MapSpec `ebpf:"go_keylog_event_tmp"` GoKeylogEvents *ebpf.MapSpec `ebpf:"go_keylog_events"` + GoKeylogEventsRingbuf *ebpf.MapSpec `ebpf:"go_keylog_events_ringbuf"` MountEventStack *ebpf.MapSpec `ebpf:"mount_event_stack"` MountEvents *ebpf.MapSpec `ebpf:"mount_events"` NatFlowMap *ebpf.MapSpec `ebpf:"nat_flow_map"` @@ -106,6 +111,7 @@ type bpf_legacyMapSpecs struct { NewNetdeviceEvents *ebpf.MapSpec `ebpf:"new_netdevice_events"` PacketEventStack *ebpf.MapSpec `ebpf:"packet_event_stack"` PacketEvents *ebpf.MapSpec `ebpf:"packet_events"` + PacketEventsRingbuf *ebpf.MapSpec `ebpf:"packet_events_ringbuf"` SockCookiePidMap *ebpf.MapSpec `ebpf:"sock_cookie_pid_map"` TidNetdeviceMap *ebpf.MapSpec `ebpf:"tid_netdevice_map"` } @@ -149,7 +155,10 @@ type bpf_legacyMaps struct { EnterMountBufs *ebpf.Map `ebpf:"enter_mount_bufs"` ExecEventStack *ebpf.Map `ebpf:"exec_event_stack"` ExecEvents *ebpf.Map `ebpf:"exec_events"` + ExecEventsRingbuf *ebpf.Map `ebpf:"exec_events_ringbuf"` + ExitEventTmp *ebpf.Map `ebpf:"exit_event_tmp"` ExitEvents *ebpf.Map `ebpf:"exit_events"` + ExitEventsRingbuf *ebpf.Map `ebpf:"exit_events_ringbuf"` FilterByKernelCount *ebpf.Map `ebpf:"filter_by_kernel_count"` FilterIfindexMap *ebpf.Map `ebpf:"filter_ifindex_map"` FilterMntnsMap *ebpf.Map `ebpf:"filter_mntns_map"` @@ -159,7 +168,9 @@ type bpf_legacyMaps struct { FilterUidMap *ebpf.Map `ebpf:"filter_uid_map"` FlowPidMap *ebpf.Map `ebpf:"flow_pid_map"` GoKeylogBufStorage *ebpf.Map `ebpf:"go_keylog_buf_storage"` + GoKeylogEventTmp *ebpf.Map `ebpf:"go_keylog_event_tmp"` GoKeylogEvents *ebpf.Map `ebpf:"go_keylog_events"` + GoKeylogEventsRingbuf *ebpf.Map `ebpf:"go_keylog_events_ringbuf"` MountEventStack *ebpf.Map `ebpf:"mount_event_stack"` MountEvents *ebpf.Map `ebpf:"mount_events"` NatFlowMap *ebpf.Map `ebpf:"nat_flow_map"` @@ -168,6 +179,7 @@ type bpf_legacyMaps struct { NewNetdeviceEvents *ebpf.Map `ebpf:"new_netdevice_events"` PacketEventStack *ebpf.Map `ebpf:"packet_event_stack"` PacketEvents *ebpf.Map `ebpf:"packet_events"` + PacketEventsRingbuf *ebpf.Map `ebpf:"packet_events_ringbuf"` SockCookiePidMap *ebpf.Map `ebpf:"sock_cookie_pid_map"` TidNetdeviceMap *ebpf.Map `ebpf:"tid_netdevice_map"` } @@ -178,7 +190,10 @@ func (m *bpf_legacyMaps) Close() error { m.EnterMountBufs, m.ExecEventStack, m.ExecEvents, + m.ExecEventsRingbuf, + m.ExitEventTmp, m.ExitEvents, + m.ExitEventsRingbuf, m.FilterByKernelCount, m.FilterIfindexMap, m.FilterMntnsMap, @@ -188,7 +203,9 @@ func (m *bpf_legacyMaps) Close() error { m.FilterUidMap, m.FlowPidMap, m.GoKeylogBufStorage, + m.GoKeylogEventTmp, m.GoKeylogEvents, + m.GoKeylogEventsRingbuf, m.MountEventStack, m.MountEvents, m.NatFlowMap, @@ -197,6 +214,7 @@ func (m *bpf_legacyMaps) Close() error { m.NewNetdeviceEvents, m.PacketEventStack, m.PacketEvents, + m.PacketEventsRingbuf, m.SockCookiePidMap, m.TidNetdeviceMap, ) diff --git a/bpf/bpf_legacy_arm64_bpfel.o b/bpf/bpf_legacy_arm64_bpfel.o index 0bc1cc76..9ae721cc 100644 Binary files a/bpf/bpf_legacy_arm64_bpfel.o and b/bpf/bpf_legacy_arm64_bpfel.o differ diff --git a/bpf/bpf_legacy_x86_bpfel.go b/bpf/bpf_legacy_x86_bpfel.go index 35b5c66e..e2638e54 100644 --- a/bpf/bpf_legacy_x86_bpfel.go +++ b/bpf/bpf_legacy_x86_bpfel.go @@ -87,7 +87,10 @@ type bpf_legacyMapSpecs struct { EnterMountBufs *ebpf.MapSpec `ebpf:"enter_mount_bufs"` ExecEventStack *ebpf.MapSpec `ebpf:"exec_event_stack"` ExecEvents *ebpf.MapSpec `ebpf:"exec_events"` + ExecEventsRingbuf *ebpf.MapSpec `ebpf:"exec_events_ringbuf"` + ExitEventTmp *ebpf.MapSpec `ebpf:"exit_event_tmp"` ExitEvents *ebpf.MapSpec `ebpf:"exit_events"` + ExitEventsRingbuf *ebpf.MapSpec `ebpf:"exit_events_ringbuf"` FilterByKernelCount *ebpf.MapSpec `ebpf:"filter_by_kernel_count"` FilterIfindexMap *ebpf.MapSpec `ebpf:"filter_ifindex_map"` FilterMntnsMap *ebpf.MapSpec `ebpf:"filter_mntns_map"` @@ -97,7 +100,9 @@ type bpf_legacyMapSpecs struct { FilterUidMap *ebpf.MapSpec `ebpf:"filter_uid_map"` FlowPidMap *ebpf.MapSpec `ebpf:"flow_pid_map"` GoKeylogBufStorage *ebpf.MapSpec `ebpf:"go_keylog_buf_storage"` + GoKeylogEventTmp *ebpf.MapSpec `ebpf:"go_keylog_event_tmp"` GoKeylogEvents *ebpf.MapSpec `ebpf:"go_keylog_events"` + GoKeylogEventsRingbuf *ebpf.MapSpec `ebpf:"go_keylog_events_ringbuf"` MountEventStack *ebpf.MapSpec `ebpf:"mount_event_stack"` MountEvents *ebpf.MapSpec `ebpf:"mount_events"` NatFlowMap *ebpf.MapSpec `ebpf:"nat_flow_map"` @@ -106,6 +111,7 @@ type bpf_legacyMapSpecs struct { NewNetdeviceEvents *ebpf.MapSpec `ebpf:"new_netdevice_events"` PacketEventStack *ebpf.MapSpec `ebpf:"packet_event_stack"` PacketEvents *ebpf.MapSpec `ebpf:"packet_events"` + PacketEventsRingbuf *ebpf.MapSpec `ebpf:"packet_events_ringbuf"` SockCookiePidMap *ebpf.MapSpec `ebpf:"sock_cookie_pid_map"` TidNetdeviceMap *ebpf.MapSpec `ebpf:"tid_netdevice_map"` } @@ -149,7 +155,10 @@ type bpf_legacyMaps struct { EnterMountBufs *ebpf.Map `ebpf:"enter_mount_bufs"` ExecEventStack *ebpf.Map `ebpf:"exec_event_stack"` ExecEvents *ebpf.Map `ebpf:"exec_events"` + ExecEventsRingbuf *ebpf.Map `ebpf:"exec_events_ringbuf"` + ExitEventTmp *ebpf.Map `ebpf:"exit_event_tmp"` ExitEvents *ebpf.Map `ebpf:"exit_events"` + ExitEventsRingbuf *ebpf.Map `ebpf:"exit_events_ringbuf"` FilterByKernelCount *ebpf.Map `ebpf:"filter_by_kernel_count"` FilterIfindexMap *ebpf.Map `ebpf:"filter_ifindex_map"` FilterMntnsMap *ebpf.Map `ebpf:"filter_mntns_map"` @@ -159,7 +168,9 @@ type bpf_legacyMaps struct { FilterUidMap *ebpf.Map `ebpf:"filter_uid_map"` FlowPidMap *ebpf.Map `ebpf:"flow_pid_map"` GoKeylogBufStorage *ebpf.Map `ebpf:"go_keylog_buf_storage"` + GoKeylogEventTmp *ebpf.Map `ebpf:"go_keylog_event_tmp"` GoKeylogEvents *ebpf.Map `ebpf:"go_keylog_events"` + GoKeylogEventsRingbuf *ebpf.Map `ebpf:"go_keylog_events_ringbuf"` MountEventStack *ebpf.Map `ebpf:"mount_event_stack"` MountEvents *ebpf.Map `ebpf:"mount_events"` NatFlowMap *ebpf.Map `ebpf:"nat_flow_map"` @@ -168,6 +179,7 @@ type bpf_legacyMaps struct { NewNetdeviceEvents *ebpf.Map `ebpf:"new_netdevice_events"` PacketEventStack *ebpf.Map `ebpf:"packet_event_stack"` PacketEvents *ebpf.Map `ebpf:"packet_events"` + PacketEventsRingbuf *ebpf.Map `ebpf:"packet_events_ringbuf"` SockCookiePidMap *ebpf.Map `ebpf:"sock_cookie_pid_map"` TidNetdeviceMap *ebpf.Map `ebpf:"tid_netdevice_map"` } @@ -178,7 +190,10 @@ func (m *bpf_legacyMaps) Close() error { m.EnterMountBufs, m.ExecEventStack, m.ExecEvents, + m.ExecEventsRingbuf, + m.ExitEventTmp, m.ExitEvents, + m.ExitEventsRingbuf, m.FilterByKernelCount, m.FilterIfindexMap, m.FilterMntnsMap, @@ -188,7 +203,9 @@ func (m *bpf_legacyMaps) Close() error { m.FilterUidMap, m.FlowPidMap, m.GoKeylogBufStorage, + m.GoKeylogEventTmp, m.GoKeylogEvents, + m.GoKeylogEventsRingbuf, m.MountEventStack, m.MountEvents, m.NatFlowMap, @@ -197,6 +214,7 @@ func (m *bpf_legacyMaps) Close() error { m.NewNetdeviceEvents, m.PacketEventStack, m.PacketEvents, + m.PacketEventsRingbuf, m.SockCookiePidMap, m.TidNetdeviceMap, ) diff --git a/bpf/bpf_legacy_x86_bpfel.o b/bpf/bpf_legacy_x86_bpfel.o index 36c67552..cb914015 100644 Binary files a/bpf/bpf_legacy_x86_bpfel.o and b/bpf/bpf_legacy_x86_bpfel.o differ diff --git a/bpf/bpf_no_tracing_arm64_bpfel.go b/bpf/bpf_no_tracing_arm64_bpfel.go index a9ef6191..a65ebe36 100644 --- a/bpf/bpf_no_tracing_arm64_bpfel.go +++ b/bpf/bpf_no_tracing_arm64_bpfel.go @@ -91,7 +91,10 @@ type bpf_no_tracingMapSpecs struct { EnterMountBufs *ebpf.MapSpec `ebpf:"enter_mount_bufs"` ExecEventStack *ebpf.MapSpec `ebpf:"exec_event_stack"` ExecEvents *ebpf.MapSpec `ebpf:"exec_events"` + ExecEventsRingbuf *ebpf.MapSpec `ebpf:"exec_events_ringbuf"` + ExitEventTmp *ebpf.MapSpec `ebpf:"exit_event_tmp"` ExitEvents *ebpf.MapSpec `ebpf:"exit_events"` + ExitEventsRingbuf *ebpf.MapSpec `ebpf:"exit_events_ringbuf"` FilterByKernelCount *ebpf.MapSpec `ebpf:"filter_by_kernel_count"` FilterIfindexMap *ebpf.MapSpec `ebpf:"filter_ifindex_map"` FilterMntnsMap *ebpf.MapSpec `ebpf:"filter_mntns_map"` @@ -101,7 +104,9 @@ type bpf_no_tracingMapSpecs struct { FilterUidMap *ebpf.MapSpec `ebpf:"filter_uid_map"` FlowPidMap *ebpf.MapSpec `ebpf:"flow_pid_map"` GoKeylogBufStorage *ebpf.MapSpec `ebpf:"go_keylog_buf_storage"` + GoKeylogEventTmp *ebpf.MapSpec `ebpf:"go_keylog_event_tmp"` GoKeylogEvents *ebpf.MapSpec `ebpf:"go_keylog_events"` + GoKeylogEventsRingbuf *ebpf.MapSpec `ebpf:"go_keylog_events_ringbuf"` MountEventStack *ebpf.MapSpec `ebpf:"mount_event_stack"` MountEvents *ebpf.MapSpec `ebpf:"mount_events"` NatFlowMap *ebpf.MapSpec `ebpf:"nat_flow_map"` @@ -110,6 +115,7 @@ type bpf_no_tracingMapSpecs struct { NewNetdeviceEvents *ebpf.MapSpec `ebpf:"new_netdevice_events"` PacketEventStack *ebpf.MapSpec `ebpf:"packet_event_stack"` PacketEvents *ebpf.MapSpec `ebpf:"packet_events"` + PacketEventsRingbuf *ebpf.MapSpec `ebpf:"packet_events_ringbuf"` SockCookiePidMap *ebpf.MapSpec `ebpf:"sock_cookie_pid_map"` TidNetdeviceMap *ebpf.MapSpec `ebpf:"tid_netdevice_map"` } @@ -154,7 +160,10 @@ type bpf_no_tracingMaps struct { EnterMountBufs *ebpf.Map `ebpf:"enter_mount_bufs"` ExecEventStack *ebpf.Map `ebpf:"exec_event_stack"` ExecEvents *ebpf.Map `ebpf:"exec_events"` + ExecEventsRingbuf *ebpf.Map `ebpf:"exec_events_ringbuf"` + ExitEventTmp *ebpf.Map `ebpf:"exit_event_tmp"` ExitEvents *ebpf.Map `ebpf:"exit_events"` + ExitEventsRingbuf *ebpf.Map `ebpf:"exit_events_ringbuf"` FilterByKernelCount *ebpf.Map `ebpf:"filter_by_kernel_count"` FilterIfindexMap *ebpf.Map `ebpf:"filter_ifindex_map"` FilterMntnsMap *ebpf.Map `ebpf:"filter_mntns_map"` @@ -164,7 +173,9 @@ type bpf_no_tracingMaps struct { FilterUidMap *ebpf.Map `ebpf:"filter_uid_map"` FlowPidMap *ebpf.Map `ebpf:"flow_pid_map"` GoKeylogBufStorage *ebpf.Map `ebpf:"go_keylog_buf_storage"` + GoKeylogEventTmp *ebpf.Map `ebpf:"go_keylog_event_tmp"` GoKeylogEvents *ebpf.Map `ebpf:"go_keylog_events"` + GoKeylogEventsRingbuf *ebpf.Map `ebpf:"go_keylog_events_ringbuf"` MountEventStack *ebpf.Map `ebpf:"mount_event_stack"` MountEvents *ebpf.Map `ebpf:"mount_events"` NatFlowMap *ebpf.Map `ebpf:"nat_flow_map"` @@ -173,6 +184,7 @@ type bpf_no_tracingMaps struct { NewNetdeviceEvents *ebpf.Map `ebpf:"new_netdevice_events"` PacketEventStack *ebpf.Map `ebpf:"packet_event_stack"` PacketEvents *ebpf.Map `ebpf:"packet_events"` + PacketEventsRingbuf *ebpf.Map `ebpf:"packet_events_ringbuf"` SockCookiePidMap *ebpf.Map `ebpf:"sock_cookie_pid_map"` TidNetdeviceMap *ebpf.Map `ebpf:"tid_netdevice_map"` } @@ -183,7 +195,10 @@ func (m *bpf_no_tracingMaps) Close() error { m.EnterMountBufs, m.ExecEventStack, m.ExecEvents, + m.ExecEventsRingbuf, + m.ExitEventTmp, m.ExitEvents, + m.ExitEventsRingbuf, m.FilterByKernelCount, m.FilterIfindexMap, m.FilterMntnsMap, @@ -193,7 +208,9 @@ func (m *bpf_no_tracingMaps) Close() error { m.FilterUidMap, m.FlowPidMap, m.GoKeylogBufStorage, + m.GoKeylogEventTmp, m.GoKeylogEvents, + m.GoKeylogEventsRingbuf, m.MountEventStack, m.MountEvents, m.NatFlowMap, @@ -202,6 +219,7 @@ func (m *bpf_no_tracingMaps) Close() error { m.NewNetdeviceEvents, m.PacketEventStack, m.PacketEvents, + m.PacketEventsRingbuf, m.SockCookiePidMap, m.TidNetdeviceMap, ) diff --git a/bpf/bpf_no_tracing_arm64_bpfel.o b/bpf/bpf_no_tracing_arm64_bpfel.o index 621450bd..ed23d2af 100644 Binary files a/bpf/bpf_no_tracing_arm64_bpfel.o and b/bpf/bpf_no_tracing_arm64_bpfel.o differ diff --git a/bpf/bpf_no_tracing_x86_bpfel.go b/bpf/bpf_no_tracing_x86_bpfel.go index 5bac9e3f..6d9ed795 100644 --- a/bpf/bpf_no_tracing_x86_bpfel.go +++ b/bpf/bpf_no_tracing_x86_bpfel.go @@ -91,7 +91,10 @@ type bpf_no_tracingMapSpecs struct { EnterMountBufs *ebpf.MapSpec `ebpf:"enter_mount_bufs"` ExecEventStack *ebpf.MapSpec `ebpf:"exec_event_stack"` ExecEvents *ebpf.MapSpec `ebpf:"exec_events"` + ExecEventsRingbuf *ebpf.MapSpec `ebpf:"exec_events_ringbuf"` + ExitEventTmp *ebpf.MapSpec `ebpf:"exit_event_tmp"` ExitEvents *ebpf.MapSpec `ebpf:"exit_events"` + ExitEventsRingbuf *ebpf.MapSpec `ebpf:"exit_events_ringbuf"` FilterByKernelCount *ebpf.MapSpec `ebpf:"filter_by_kernel_count"` FilterIfindexMap *ebpf.MapSpec `ebpf:"filter_ifindex_map"` FilterMntnsMap *ebpf.MapSpec `ebpf:"filter_mntns_map"` @@ -101,7 +104,9 @@ type bpf_no_tracingMapSpecs struct { FilterUidMap *ebpf.MapSpec `ebpf:"filter_uid_map"` FlowPidMap *ebpf.MapSpec `ebpf:"flow_pid_map"` GoKeylogBufStorage *ebpf.MapSpec `ebpf:"go_keylog_buf_storage"` + GoKeylogEventTmp *ebpf.MapSpec `ebpf:"go_keylog_event_tmp"` GoKeylogEvents *ebpf.MapSpec `ebpf:"go_keylog_events"` + GoKeylogEventsRingbuf *ebpf.MapSpec `ebpf:"go_keylog_events_ringbuf"` MountEventStack *ebpf.MapSpec `ebpf:"mount_event_stack"` MountEvents *ebpf.MapSpec `ebpf:"mount_events"` NatFlowMap *ebpf.MapSpec `ebpf:"nat_flow_map"` @@ -110,6 +115,7 @@ type bpf_no_tracingMapSpecs struct { NewNetdeviceEvents *ebpf.MapSpec `ebpf:"new_netdevice_events"` PacketEventStack *ebpf.MapSpec `ebpf:"packet_event_stack"` PacketEvents *ebpf.MapSpec `ebpf:"packet_events"` + PacketEventsRingbuf *ebpf.MapSpec `ebpf:"packet_events_ringbuf"` SockCookiePidMap *ebpf.MapSpec `ebpf:"sock_cookie_pid_map"` TidNetdeviceMap *ebpf.MapSpec `ebpf:"tid_netdevice_map"` } @@ -154,7 +160,10 @@ type bpf_no_tracingMaps struct { EnterMountBufs *ebpf.Map `ebpf:"enter_mount_bufs"` ExecEventStack *ebpf.Map `ebpf:"exec_event_stack"` ExecEvents *ebpf.Map `ebpf:"exec_events"` + ExecEventsRingbuf *ebpf.Map `ebpf:"exec_events_ringbuf"` + ExitEventTmp *ebpf.Map `ebpf:"exit_event_tmp"` ExitEvents *ebpf.Map `ebpf:"exit_events"` + ExitEventsRingbuf *ebpf.Map `ebpf:"exit_events_ringbuf"` FilterByKernelCount *ebpf.Map `ebpf:"filter_by_kernel_count"` FilterIfindexMap *ebpf.Map `ebpf:"filter_ifindex_map"` FilterMntnsMap *ebpf.Map `ebpf:"filter_mntns_map"` @@ -164,7 +173,9 @@ type bpf_no_tracingMaps struct { FilterUidMap *ebpf.Map `ebpf:"filter_uid_map"` FlowPidMap *ebpf.Map `ebpf:"flow_pid_map"` GoKeylogBufStorage *ebpf.Map `ebpf:"go_keylog_buf_storage"` + GoKeylogEventTmp *ebpf.Map `ebpf:"go_keylog_event_tmp"` GoKeylogEvents *ebpf.Map `ebpf:"go_keylog_events"` + GoKeylogEventsRingbuf *ebpf.Map `ebpf:"go_keylog_events_ringbuf"` MountEventStack *ebpf.Map `ebpf:"mount_event_stack"` MountEvents *ebpf.Map `ebpf:"mount_events"` NatFlowMap *ebpf.Map `ebpf:"nat_flow_map"` @@ -173,6 +184,7 @@ type bpf_no_tracingMaps struct { NewNetdeviceEvents *ebpf.Map `ebpf:"new_netdevice_events"` PacketEventStack *ebpf.Map `ebpf:"packet_event_stack"` PacketEvents *ebpf.Map `ebpf:"packet_events"` + PacketEventsRingbuf *ebpf.Map `ebpf:"packet_events_ringbuf"` SockCookiePidMap *ebpf.Map `ebpf:"sock_cookie_pid_map"` TidNetdeviceMap *ebpf.Map `ebpf:"tid_netdevice_map"` } @@ -183,7 +195,10 @@ func (m *bpf_no_tracingMaps) Close() error { m.EnterMountBufs, m.ExecEventStack, m.ExecEvents, + m.ExecEventsRingbuf, + m.ExitEventTmp, m.ExitEvents, + m.ExitEventsRingbuf, m.FilterByKernelCount, m.FilterIfindexMap, m.FilterMntnsMap, @@ -193,7 +208,9 @@ func (m *bpf_no_tracingMaps) Close() error { m.FilterUidMap, m.FlowPidMap, m.GoKeylogBufStorage, + m.GoKeylogEventTmp, m.GoKeylogEvents, + m.GoKeylogEventsRingbuf, m.MountEventStack, m.MountEvents, m.NatFlowMap, @@ -202,6 +219,7 @@ func (m *bpf_no_tracingMaps) Close() error { m.NewNetdeviceEvents, m.PacketEventStack, m.PacketEvents, + m.PacketEventsRingbuf, m.SockCookiePidMap, m.TidNetdeviceMap, ) diff --git a/bpf/bpf_no_tracing_x86_bpfel.o b/bpf/bpf_no_tracing_x86_bpfel.o index 16e8d0a8..66f95b33 100644 Binary files a/bpf/bpf_no_tracing_x86_bpfel.o and b/bpf/bpf_no_tracing_x86_bpfel.o differ diff --git a/bpf/bpf_x86_bpfel.go b/bpf/bpf_x86_bpfel.go index 0905e45f..7ad63e3c 100644 --- a/bpf/bpf_x86_bpfel.go +++ b/bpf/bpf_x86_bpfel.go @@ -42,6 +42,8 @@ type BpfGconfigT struct { FilterComm [16]int8 FilterCommEnable uint8 FilterIfindexEnable uint8 + UseRingbufSubmitSkb uint8 + _ [3]byte MaxPayloadSize uint32 } @@ -211,7 +213,10 @@ type BpfMapSpecs struct { EnterMountBufs *ebpf.MapSpec `ebpf:"enter_mount_bufs"` ExecEventStack *ebpf.MapSpec `ebpf:"exec_event_stack"` ExecEvents *ebpf.MapSpec `ebpf:"exec_events"` + ExecEventsRingbuf *ebpf.MapSpec `ebpf:"exec_events_ringbuf"` + ExitEventTmp *ebpf.MapSpec `ebpf:"exit_event_tmp"` ExitEvents *ebpf.MapSpec `ebpf:"exit_events"` + ExitEventsRingbuf *ebpf.MapSpec `ebpf:"exit_events_ringbuf"` FilterByKernelCount *ebpf.MapSpec `ebpf:"filter_by_kernel_count"` FilterIfindexMap *ebpf.MapSpec `ebpf:"filter_ifindex_map"` FilterMntnsMap *ebpf.MapSpec `ebpf:"filter_mntns_map"` @@ -221,7 +226,9 @@ type BpfMapSpecs struct { FilterUidMap *ebpf.MapSpec `ebpf:"filter_uid_map"` FlowPidMap *ebpf.MapSpec `ebpf:"flow_pid_map"` GoKeylogBufStorage *ebpf.MapSpec `ebpf:"go_keylog_buf_storage"` + GoKeylogEventTmp *ebpf.MapSpec `ebpf:"go_keylog_event_tmp"` GoKeylogEvents *ebpf.MapSpec `ebpf:"go_keylog_events"` + GoKeylogEventsRingbuf *ebpf.MapSpec `ebpf:"go_keylog_events_ringbuf"` MountEventStack *ebpf.MapSpec `ebpf:"mount_event_stack"` MountEvents *ebpf.MapSpec `ebpf:"mount_events"` NatFlowMap *ebpf.MapSpec `ebpf:"nat_flow_map"` @@ -230,6 +237,7 @@ type BpfMapSpecs struct { NewNetdeviceEvents *ebpf.MapSpec `ebpf:"new_netdevice_events"` PacketEventStack *ebpf.MapSpec `ebpf:"packet_event_stack"` PacketEvents *ebpf.MapSpec `ebpf:"packet_events"` + PacketEventsRingbuf *ebpf.MapSpec `ebpf:"packet_events_ringbuf"` SockCookiePidMap *ebpf.MapSpec `ebpf:"sock_cookie_pid_map"` TidNetdeviceMap *ebpf.MapSpec `ebpf:"tid_netdevice_map"` } @@ -274,7 +282,10 @@ type BpfMaps struct { EnterMountBufs *ebpf.Map `ebpf:"enter_mount_bufs"` ExecEventStack *ebpf.Map `ebpf:"exec_event_stack"` ExecEvents *ebpf.Map `ebpf:"exec_events"` + ExecEventsRingbuf *ebpf.Map `ebpf:"exec_events_ringbuf"` + ExitEventTmp *ebpf.Map `ebpf:"exit_event_tmp"` ExitEvents *ebpf.Map `ebpf:"exit_events"` + ExitEventsRingbuf *ebpf.Map `ebpf:"exit_events_ringbuf"` FilterByKernelCount *ebpf.Map `ebpf:"filter_by_kernel_count"` FilterIfindexMap *ebpf.Map `ebpf:"filter_ifindex_map"` FilterMntnsMap *ebpf.Map `ebpf:"filter_mntns_map"` @@ -284,7 +295,9 @@ type BpfMaps struct { FilterUidMap *ebpf.Map `ebpf:"filter_uid_map"` FlowPidMap *ebpf.Map `ebpf:"flow_pid_map"` GoKeylogBufStorage *ebpf.Map `ebpf:"go_keylog_buf_storage"` + GoKeylogEventTmp *ebpf.Map `ebpf:"go_keylog_event_tmp"` GoKeylogEvents *ebpf.Map `ebpf:"go_keylog_events"` + GoKeylogEventsRingbuf *ebpf.Map `ebpf:"go_keylog_events_ringbuf"` MountEventStack *ebpf.Map `ebpf:"mount_event_stack"` MountEvents *ebpf.Map `ebpf:"mount_events"` NatFlowMap *ebpf.Map `ebpf:"nat_flow_map"` @@ -293,6 +306,7 @@ type BpfMaps struct { NewNetdeviceEvents *ebpf.Map `ebpf:"new_netdevice_events"` PacketEventStack *ebpf.Map `ebpf:"packet_event_stack"` PacketEvents *ebpf.Map `ebpf:"packet_events"` + PacketEventsRingbuf *ebpf.Map `ebpf:"packet_events_ringbuf"` SockCookiePidMap *ebpf.Map `ebpf:"sock_cookie_pid_map"` TidNetdeviceMap *ebpf.Map `ebpf:"tid_netdevice_map"` } @@ -303,7 +317,10 @@ func (m *BpfMaps) Close() error { m.EnterMountBufs, m.ExecEventStack, m.ExecEvents, + m.ExecEventsRingbuf, + m.ExitEventTmp, m.ExitEvents, + m.ExitEventsRingbuf, m.FilterByKernelCount, m.FilterIfindexMap, m.FilterMntnsMap, @@ -313,7 +330,9 @@ func (m *BpfMaps) Close() error { m.FilterUidMap, m.FlowPidMap, m.GoKeylogBufStorage, + m.GoKeylogEventTmp, m.GoKeylogEvents, + m.GoKeylogEventsRingbuf, m.MountEventStack, m.MountEvents, m.NatFlowMap, @@ -322,6 +341,7 @@ func (m *BpfMaps) Close() error { m.NewNetdeviceEvents, m.PacketEventStack, m.PacketEvents, + m.PacketEventsRingbuf, m.SockCookiePidMap, m.TidNetdeviceMap, ) diff --git a/bpf/bpf_x86_bpfel.o b/bpf/bpf_x86_bpfel.o index 12594af2..d751595f 100644 Binary files a/bpf/bpf_x86_bpfel.o and b/bpf/bpf_x86_bpfel.o differ diff --git a/bpf/event.go b/bpf/event.go index 12f220a7..4badc60e 100644 --- a/bpf/event.go +++ b/bpf/event.go @@ -12,40 +12,64 @@ import ( "unsafe" "github.com/cilium/ebpf/perf" + "github.com/cilium/ebpf/ringbuf" "github.com/mozillazg/ptcpdump/internal/log" ) +type EventReader struct { + perfReader *perf.Reader + ringbufReader *ringbuf.Reader +} + +type EventRecord struct { + RawSample []byte + LostSamples uint64 +} + type BpfPacketEventWithPayloadT struct { BpfPacketEventT Payload []byte } func (b *BPF) PullPacketEvents(ctx context.Context, chanSize int, maxPacketSize int) (<-chan BpfPacketEventWithPayloadT, error) { - pageSize := os.Getpagesize() - log.Infof("pagesize is %d", pageSize) - perCPUBuffer := pageSize * 64 - eventSize := int(unsafe.Sizeof(BpfPacketEventT{})) + maxPacketSize - if eventSize >= perCPUBuffer { - perCPUBuffer = perCPUBuffer * (1 + (eventSize / perCPUBuffer)) - } - log.Infof("use %d as perCPUBuffer", perCPUBuffer) + var reader EventReader + if b.supportRingBuf && b.useRingBufSubmitSkb { + log.Info("use ringbuf for packet events") + ringbufReader, err := ringbuf.NewReader(b.objs.PacketEventsRingbuf) + if err != nil { + return nil, fmt.Errorf(": %w", err) + } + reader.ringbufReader = ringbufReader + } else { + log.Info("use perf for packet events") + pageSize := os.Getpagesize() + log.Infof("pagesize is %d", pageSize) + perCPUBuffer := pageSize * 64 + eventSize := int(unsafe.Sizeof(BpfPacketEventT{})) + maxPacketSize + if eventSize >= perCPUBuffer { + perCPUBuffer = perCPUBuffer * (1 + (eventSize / perCPUBuffer)) + } + log.Infof("use %d as perCPUBuffer", perCPUBuffer) - reader, err := perf.NewReader(b.objs.PacketEvents, perCPUBuffer) - if err != nil { - return nil, fmt.Errorf(": %w", err) + preader, err := perf.NewReader(b.objs.PacketEvents, perCPUBuffer) + if err != nil { + return nil, fmt.Errorf(": %w", err) + } + reader.perfReader = preader } + ch := make(chan BpfPacketEventWithPayloadT, chanSize) go func() { defer close(ch) defer reader.Close() - b.handlePacketEvents(ctx, reader, ch) + b.handlePacketEvents(ctx, &reader, ch) }() return ch, nil } -func (b *BPF) handlePacketEvents(ctx context.Context, reader *perf.Reader, ch chan<- BpfPacketEventWithPayloadT) { +func (b *BPF) handlePacketEvents(ctx context.Context, reader *EventReader, ch chan<- BpfPacketEventWithPayloadT) { for { select { case <-ctx.Done(): @@ -55,7 +79,7 @@ func (b *BPF) handlePacketEvents(ctx context.Context, reader *perf.Reader, ch ch record, err := reader.Read() if err != nil { - if errors.Is(err, perf.ErrClosed) { + if errors.Is(err, perf.ErrClosed) || errors.Is(err, ringbuf.ErrClosed) { return } if errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) { @@ -82,36 +106,49 @@ func parsePacketEvent(rawSample []byte) (*BpfPacketEventWithPayloadT, error) { if err := binary.Read(bytes.NewBuffer(rawSample), binary.LittleEndian, &event.Meta); err != nil { return nil, fmt.Errorf("parse meta: %w", err) } - event.Payload = make([]byte, int(event.Meta.PacketSize)) + event.Payload = make([]byte, int(event.Meta.PayloadLen)) copy(event.Payload[:], rawSample[unsafe.Sizeof(BpfPacketEventT{}):]) return &event, nil } func (b *BPF) PullExecEvents(ctx context.Context, chanSize int) (<-chan BpfExecEventT, error) { - pageSize := os.Getpagesize() - log.Infof("pagesize is %d", pageSize) - perCPUBuffer := pageSize * 64 - eventSize := int(unsafe.Sizeof(BpfExecEventT{})) - if eventSize >= perCPUBuffer { - perCPUBuffer = perCPUBuffer * (1 + (eventSize / perCPUBuffer)) - } - log.Infof("use %d as perCPUBuffer", perCPUBuffer) + var reader EventReader - reader, err := perf.NewReader(b.objs.ExecEvents, perCPUBuffer) - if err != nil { - return nil, fmt.Errorf(": %w", err) + if b.supportRingBuf { + log.Info("use ringbuf for exec events") + ringbufReader, err := ringbuf.NewReader(b.objs.ExecEventsRingbuf) + if err != nil { + return nil, fmt.Errorf(": %w", err) + } + reader.ringbufReader = ringbufReader + } else { + log.Info("use perf for exec events") + pageSize := os.Getpagesize() + log.Infof("pagesize is %d", pageSize) + perCPUBuffer := pageSize * 64 + eventSize := int(unsafe.Sizeof(BpfExecEventT{})) + if eventSize >= perCPUBuffer { + perCPUBuffer = perCPUBuffer * (1 + (eventSize / perCPUBuffer)) + } + log.Infof("use %d as perCPUBuffer", perCPUBuffer) + + preader, err := perf.NewReader(b.objs.ExecEvents, perCPUBuffer) + if err != nil { + return nil, fmt.Errorf(": %w", err) + } + reader.perfReader = preader } ch := make(chan BpfExecEventT, chanSize) go func() { defer close(ch) defer reader.Close() - b.handleExecEvents(ctx, reader, ch) + b.handleExecEvents(ctx, &reader, ch) }() return ch, nil } -func (b *BPF) handleExecEvents(ctx context.Context, reader *perf.Reader, ch chan<- BpfExecEventT) { +func (b *BPF) handleExecEvents(ctx context.Context, reader *EventReader, ch chan<- BpfExecEventT) { for { select { case <-ctx.Done(): @@ -121,7 +158,7 @@ func (b *BPF) handleExecEvents(ctx context.Context, reader *perf.Reader, ch chan record, err := reader.Read() if err != nil { - if errors.Is(err, perf.ErrClosed) { + if errors.Is(err, perf.ErrClosed) || errors.Is(err, ringbuf.ErrClosed) { return } if errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) { @@ -144,30 +181,44 @@ func (b *BPF) handleExecEvents(ctx context.Context, reader *perf.Reader, ch chan } func (b *BPF) PullGoKeyLogEvents(ctx context.Context, chanSize int) (<-chan BpfGoKeylogEventT, error) { - pageSize := os.Getpagesize() - log.Infof("pagesize is %d", pageSize) - perCPUBuffer := pageSize * 4 - eventSize := int(unsafe.Sizeof(BpfGoKeylogEventT{})) - if eventSize >= perCPUBuffer { - perCPUBuffer = perCPUBuffer * (1 + (eventSize / perCPUBuffer)) - } - log.Infof("use %d as perCPUBuffer", perCPUBuffer) + var reader EventReader - reader, err := perf.NewReader(b.objs.GoKeylogEvents, perCPUBuffer) - if err != nil { - return nil, fmt.Errorf(": %w", err) + if b.supportRingBuf { + log.Info("use ringbuf for go keylog events") + ringbufReader, err := ringbuf.NewReader(b.objs.GoKeylogEventsRingbuf) + if err != nil { + return nil, fmt.Errorf(": %w", err) + } + reader.ringbufReader = ringbufReader + } else { + log.Info("use perf for go keylog events") + pageSize := os.Getpagesize() + log.Infof("pagesize is %d", pageSize) + perCPUBuffer := pageSize * 4 + eventSize := int(unsafe.Sizeof(BpfGoKeylogEventT{})) + if eventSize >= perCPUBuffer { + perCPUBuffer = perCPUBuffer * (1 + (eventSize / perCPUBuffer)) + } + log.Infof("use %d as perCPUBuffer", perCPUBuffer) + + preader, err := perf.NewReader(b.objs.GoKeylogEvents, perCPUBuffer) + if err != nil { + return nil, fmt.Errorf(": %w", err) + } + reader.perfReader = preader } + ch := make(chan BpfGoKeylogEventT, chanSize) go func() { defer close(ch) defer reader.Close() - b.handleGoKeyLogEvents(ctx, reader, ch) + b.handleGoKeyLogEvents(ctx, &reader, ch) }() return ch, nil } -func (b *BPF) handleGoKeyLogEvents(ctx context.Context, reader *perf.Reader, ch chan<- BpfGoKeylogEventT) { +func (b *BPF) handleGoKeyLogEvents(ctx context.Context, reader *EventReader, ch chan<- BpfGoKeylogEventT) { for { select { case <-ctx.Done(): @@ -177,7 +228,7 @@ func (b *BPF) handleGoKeyLogEvents(ctx context.Context, reader *perf.Reader, ch record, err := reader.Read() if err != nil { - if errors.Is(err, perf.ErrClosed) { + if errors.Is(err, perf.ErrClosed) || errors.Is(err, ringbuf.ErrClosed) { return } if errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) { @@ -233,7 +284,7 @@ func (b *BPF) handleNewNetDeviceEvents(ctx context.Context, reader *perf.Reader, record, err := reader.Read() if err != nil { - if errors.Is(err, perf.ErrClosed) { + if errors.Is(err, perf.ErrClosed) || errors.Is(err, ringbuf.ErrClosed) { return } if errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) { @@ -300,7 +351,7 @@ func (b *BPF) handleNetDeviceChangeEvents(ctx context.Context, reader *perf.Read record, err := reader.Read() if err != nil { - if errors.Is(err, perf.ErrClosed) { + if errors.Is(err, perf.ErrClosed) || errors.Is(err, ringbuf.ErrClosed) { return } if errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) { @@ -369,7 +420,7 @@ func (b *BPF) handleMountEvents(ctx context.Context, reader *perf.Reader, ch cha record, err := reader.Read() if err != nil { - if errors.Is(err, perf.ErrClosed) { + if errors.Is(err, perf.ErrClosed) || errors.Is(err, ringbuf.ErrClosed) { return } if errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) { @@ -418,30 +469,44 @@ func parseExecEvent(rawSample []byte) (*BpfExecEventT, error) { } func (b *BPF) PullExitEvents(ctx context.Context, chanSize int) (<-chan BpfExitEventT, error) { - pageSize := os.Getpagesize() - log.Infof("pagesize is %d", pageSize) - perCPUBuffer := pageSize * 4 - eventSize := int(unsafe.Sizeof(BpfExitEventT{})) - if eventSize >= perCPUBuffer { - perCPUBuffer = perCPUBuffer * (1 + (eventSize / perCPUBuffer)) - } - log.Infof("use %d as perCPUBuffer", perCPUBuffer) + var reader EventReader + if b.supportRingBuf { + log.Info("use ringbuf for exit events") + ringbufReader, err := ringbuf.NewReader(b.objs.ExitEventsRingbuf) + if err != nil { + return nil, fmt.Errorf(": %w", err) + } + reader.ringbufReader = ringbufReader + } else { + log.Info("use perf for exit events") + + pageSize := os.Getpagesize() + log.Infof("pagesize is %d", pageSize) + perCPUBuffer := pageSize * 4 + eventSize := int(unsafe.Sizeof(BpfExitEventT{})) + if eventSize >= perCPUBuffer { + perCPUBuffer = perCPUBuffer * (1 + (eventSize / perCPUBuffer)) + } + log.Infof("use %d as perCPUBuffer", perCPUBuffer) - reader, err := perf.NewReader(b.objs.ExitEvents, perCPUBuffer) - if err != nil { - return nil, fmt.Errorf(": %w", err) + preader, err := perf.NewReader(b.objs.ExitEvents, perCPUBuffer) + if err != nil { + return nil, fmt.Errorf(": %w", err) + } + reader.perfReader = preader } + ch := make(chan BpfExitEventT, chanSize) go func() { defer close(ch) defer reader.Close() - b.handleExitEvents(ctx, reader, ch) + b.handleExitEvents(ctx, &reader, ch) }() return ch, nil } -func (b *BPF) handleExitEvents(ctx context.Context, reader *perf.Reader, ch chan<- BpfExitEventT) { +func (b *BPF) handleExitEvents(ctx context.Context, reader *EventReader, ch chan<- BpfExitEventT) { for { select { case <-ctx.Done(): @@ -451,7 +516,7 @@ func (b *BPF) handleExitEvents(ctx context.Context, reader *perf.Reader, ch chan record, err := reader.Read() if err != nil { - if errors.Is(err, perf.ErrClosed) { + if errors.Is(err, perf.ErrClosed) || errors.Is(err, ringbuf.ErrClosed) { return } if errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) { @@ -480,3 +545,44 @@ func parseExitEvent(rawSample []byte) (*BpfExitEventT, error) { } return &event, nil } + +func NewEventReader(perfReader *perf.Reader, ringbufReader *ringbuf.Reader) *EventReader { + return &EventReader{ + perfReader: perfReader, + ringbufReader: ringbufReader, + } +} + +func (r *EventReader) Read() (*EventRecord, error) { + if r.perfReader != nil { + record, err := r.perfReader.Read() + if err != nil { + return nil, err + } + return &EventRecord{ + RawSample: record.RawSample, + LostSamples: record.LostSamples, + }, nil + } + if r.ringbufReader != nil { + record, err := r.ringbufReader.Read() + if err != nil { + return nil, err + } + return &EventRecord{ + RawSample: record.RawSample, + LostSamples: 0, + }, nil + } + return nil, errors.New("no reader") +} + +func (r *EventReader) Close() error { + if r.perfReader != nil { + return r.perfReader.Close() + } + if r.ringbufReader != nil { + return r.ringbufReader.Close() + } + return nil +} diff --git a/bpf/gotls.h b/bpf/gotls.h index 96e0a9a1..d610db89 100644 --- a/bpf/gotls.h +++ b/bpf/gotls.h @@ -44,6 +44,18 @@ struct { __uint(value_size, sizeof(u32)); } go_keylog_events SEC(".maps"); +struct { + __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); + __uint(max_entries, 1); + __uint(key_size, sizeof(u32)); + __uint(value_size, sizeof(struct go_keylog_event_t)); +} go_keylog_event_tmp SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_RINGBUF); + __uint(max_entries, 1 << 24); +} go_keylog_events_ringbuf SEC(".maps"); + const struct go_keylog_event_t *unused7 __attribute__((unused)); SEC("uprobe/go:crypto/tls.(*Config).writeKeyLog") @@ -67,8 +79,10 @@ int uprobe__go_builtin__tls__write_key_log(struct pt_regs *ctx) { SEC("uprobe/go:crypto/tls.(*Config).writeKeyLog/ret") int uprobe__go_builtin__tls__write_key_log__ret(struct pt_regs *ctx) { struct go_keylog_buf_t *buf; - struct go_keylog_event_t event = {0}; + struct go_keylog_event_t *event; int ret; + u32 u32_zero = 0; + bool use_ringbuf = false; u32 smp_id = bpf_get_smp_processor_id(); buf = bpf_map_lookup_elem(&go_keylog_buf_storage, &smp_id); @@ -77,32 +91,50 @@ int uprobe__go_builtin__tls__write_key_log__ret(struct pt_regs *ctx) { return 0; } - bpf_probe_read_kernel(&event.label_len, sizeof(event.label_len), &(buf->label_len_ptr)); - bpf_probe_read_kernel(&event.client_random_len, sizeof(event.client_random_len), &(buf->random_len_ptr)); - bpf_probe_read_kernel(&event.secret_len, sizeof(event.secret_len), &(buf->secret_len_ptr)); - if (event.label_len == 0 && event.client_random_len == 0 && event.secret_len == 0) { + if (ringbuf_available()) { + event = bpf_ringbuf_reserve(&go_keylog_events_ringbuf, sizeof(*event), 0); + use_ringbuf = true; + } else { + event = bpf_map_lookup_elem(&go_keylog_event_tmp, &u32_zero); + } + if (!event) { + return 0; + } + + bpf_probe_read_kernel(&event->label_len, sizeof(event->label_len), &(buf->label_len_ptr)); + bpf_probe_read_kernel(&event->client_random_len, sizeof(event->client_random_len), &(buf->random_len_ptr)); + bpf_probe_read_kernel(&event->secret_len, sizeof(event->secret_len), &(buf->secret_len_ptr)); + if (event->label_len == 0 && event->client_random_len == 0 && event->secret_len == 0) { // debug_log("go tls read filed, label_len: %d, client_random_len: %d, secret_len: %d", - // event.label_len, event.client_random_len, event.secret_len ); + // event->label_len, event->client_random_len, event->secret_len ); + if (use_ringbuf) { + bpf_ringbuf_discard(event, 0); + } return 0; } - ret = bpf_probe_read_user(&event.label, sizeof(event.label), (void *)(buf->label_ptr)); + ret = bpf_probe_read_user(&event->label, sizeof(event->label), (void *)(buf->label_ptr)); if (ret < 0) { // debug_log("go labels, ret: %d", ret); } - ret = bpf_probe_read_user(&event.client_random, sizeof(event.client_random), (void *)(buf->random_ptr)); + ret = bpf_probe_read_user(&event->client_random, sizeof(event->client_random), (void *)(buf->random_ptr)); if (ret < 0) { // debug_log("go random, ret: %d", ret); } - ret = bpf_probe_read_user(&event.secret, sizeof(event.secret), (void *)(buf->secret_ptr)); + ret = bpf_probe_read_user(&event->secret, sizeof(event->secret), (void *)(buf->secret_ptr)); if (ret < 0) { // debug_log("go secret, ret: %d", ret); } - // debug_log("go label_len: %d, client_random_len: %d, secret_len: %d", event.label_len, - // event.client_random_len, event.secret_len); - // debug_log("go label: %x, client_random: %x, secret: %x", event.label, - // event.client_random, event.secret); - ret = bpf_perf_event_output(ctx, &go_keylog_events, BPF_F_CURRENT_CPU, &event, sizeof(event)); + // debug_log("go label_len: %d, client_random_len: %d, secret_len: %d", event->label_len, + // event->client_random_len, event->secret_len); + // debug_log("go label: %x, client_random: %x, secret: %x", event->label, + // event->client_random, event->secret); + + if (use_ringbuf) { + bpf_ringbuf_submit(event, 0); + } else { + ret = bpf_perf_event_output(ctx, &go_keylog_events, BPF_F_CURRENT_CPU, event, sizeof(*event)); + } if (ret < 0) { // debug_log("go tls: per event failed, %d", ret); } diff --git a/bpf/headers/compat.h b/bpf/headers/compat.h index e623d5f2..2eef2f55 100644 --- a/bpf/headers/compat.h +++ b/bpf/headers/compat.h @@ -2,6 +2,10 @@ #define __PTCPDUMP_COMPAT_H__ #include "vmlinux.h" +#include +#include +#include +#include #define TASK_COMM_LEN 16 @@ -11,6 +15,7 @@ struct gconfig_t { char filter_comm[TASK_COMM_LEN]; u8 filter_comm_enable; u8 filter_ifindex_enable; + u8 use_ringbuf_submit_skb; u32 max_payload_size; }; @@ -39,4 +44,16 @@ static const u32 u32_zero = 0; #define GET_CONFIG() #endif +#ifdef LEGACY_KERNEL +static __always_inline bool ringbuf_available() { return false; } +#else + +static __always_inline bool ringbuf_available() { + if (bpf_core_type_exists(struct bpf_ringbuf)) { + return true; + } + return false; +} +#endif + #endif /* __PTCPDUMP_COMPAT_H__ */ diff --git a/bpf/process.h b/bpf/process.h index ab972225..f9567d6e 100644 --- a/bpf/process.h +++ b/bpf/process.h @@ -89,18 +89,35 @@ struct { __type(value, struct exec_event_t); } exec_event_stack SEC(".maps"); +struct { + __uint(type, BPF_MAP_TYPE_RINGBUF); + __uint(max_entries, 1 << 24); +} exec_events_ringbuf SEC(".maps"); + struct { __uint(type, BPF_MAP_TYPE_PERF_EVENT_ARRAY); __uint(key_size, sizeof(u32)); __uint(value_size, sizeof(u32)); } exec_events SEC(".maps"); +struct { + __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); + __uint(max_entries, 1); + __type(key, u32); + __type(value, struct exit_event_t); +} exit_event_tmp SEC(".maps"); + struct { __uint(type, BPF_MAP_TYPE_PERF_EVENT_ARRAY); __uint(key_size, sizeof(u32)); __uint(value_size, sizeof(u32)); } exit_events SEC(".maps"); +struct { + __uint(type, BPF_MAP_TYPE_RINGBUF); + __uint(max_entries, 1 << 17); +} exit_events_ringbuf SEC(".maps"); + const struct exec_event_t *unused2 __attribute__((unused)); const struct process_meta_t *unused4 __attribute__((unused)); const struct exit_event_t *unused5 __attribute__((unused)); @@ -286,12 +303,19 @@ static __always_inline void handle_exec(void *ctx, struct task_struct *task, pid // if (process_filter(task) < 0) { // return; // } + bool use_ringbuf = false; struct exec_event_t *event; #ifdef LEGACY_KERNEL u32 u32_zero = 0; #endif - event = bpf_map_lookup_elem(&exec_event_stack, &u32_zero); + + if (ringbuf_available()) { + event = bpf_ringbuf_reserve(&exec_events_ringbuf, sizeof(*event), 0); + use_ringbuf = true; + } else { + event = bpf_map_lookup_elem(&exec_event_stack, &u32_zero); + } if (!event) { // debug_log("[ptcpdump] exec_event_stack failed\n"); return; @@ -323,9 +347,13 @@ static __always_inline void handle_exec(void *ctx, struct task_struct *task, pid event->args_size = arg_length; } - int event_ret = bpf_perf_event_output(ctx, &exec_events, BPF_F_CURRENT_CPU, event, sizeof(*event)); - if (event_ret != 0) { - // debug_log("[ptcpdump] bpf_perf_event_output exec_events failed: %d\n", event_ret); + if (use_ringbuf) { + bpf_ringbuf_submit(event, 0); + } else { + int event_ret = bpf_perf_event_output(ctx, &exec_events, BPF_F_CURRENT_CPU, event, sizeof(*event)); + if (event_ret != 0) { + // debug_log("[ptcpdump] bpf_perf_event_output exec_events failed: %d\n", event_ret); + } } return; } @@ -341,12 +369,28 @@ static __always_inline void handle_exit(void *ctx, struct task_struct *task) { bpf_map_delete_elem(&filter_pid_map, &pid); } - struct exit_event_t event = { - .pid = pid, - }; - int event_ret = bpf_perf_event_output(ctx, &exit_events, BPF_F_CURRENT_CPU, &event, sizeof(event)); - if (event_ret != 0) { - // debug_log("[ptcpdump] bpf_perf_event_output exit_events failed: %d\n", event_ret); + struct exit_event_t *event; + bool use_ringbuf = false; + + if (ringbuf_available()) { + event = bpf_ringbuf_reserve(&exit_events_ringbuf, sizeof(*event), 0); + use_ringbuf = true; + } else { + event = bpf_map_lookup_elem(&exit_event_tmp, &pid); + } + if (!event) { + return; + } + + event->pid = pid; + + if (use_ringbuf) { + bpf_ringbuf_submit(event, 0); + } else { + int event_ret = bpf_perf_event_output(ctx, &exit_events, BPF_F_CURRENT_CPU, event, sizeof(*event)); + if (event_ret != 0) { + // debug_log("[ptcpdump] bpf_perf_event_output exit_events failed: %d\n", event_ret); + } } return; diff --git a/bpf/ptcpdump.c b/bpf/ptcpdump.c index 56fca60a..0553b0de 100644 --- a/bpf/ptcpdump.c +++ b/bpf/ptcpdump.c @@ -76,6 +76,15 @@ struct { __uint(value_size, sizeof(u32)); } packet_events SEC(".maps"); +struct skb_data_t { + u8 data[1 << 18]; +}; + +struct { + __uint(type, BPF_MAP_TYPE_RINGBUF); + __uint(max_entries, 1 << 25); +} packet_events_ringbuf SEC(".maps"); + struct { __uint(type, BPF_MAP_TYPE_HASH); __uint(max_entries, 1); @@ -395,19 +404,37 @@ static __always_inline void handle_tc(bool cgroup_skb, struct __sk_buff *skb, bo #ifdef LEGACY_KERNEL u32 u32_zero = 0; #endif + GET_CONFIG() + void *ringbuf = NULL; struct packet_event_t *event; - event = bpf_map_lookup_elem(&packet_event_stack, &u32_zero); + bool use_ringbuf = false; + + if (g.use_ringbuf_submit_skb && ringbuf_available()) { + ringbuf = bpf_ringbuf_reserve(&packet_events_ringbuf, sizeof(*event) + g.max_payload_size, 0); + if (!ringbuf) { + return; + } + event = (struct packet_event_t *)ringbuf; + use_ringbuf = true; + } else { + event = bpf_map_lookup_elem(&packet_event_stack, &u32_zero); + } + if (!event) { // debug_log("[ptcpdump] packet_event_stack failed\n"); return; } + __builtin_memset(&event->meta, 0, sizeof(event->meta)); __builtin_memset(&event->meta.process, 0, sizeof(event->meta.process)); __builtin_memset(&event->meta.process.cgroup_name, 0, sizeof(event->meta.process.cgroup_name)); if (fill_packet_event_meta(skb, cgroup_skb, &event->meta, egress) < 0) { // debug_log("tc, not found pid\n"); + if (use_ringbuf) { + bpf_ringbuf_discard(ringbuf, 0); + } return; }; // if (process_meta_filter(&event->meta.process) < 0) { @@ -434,8 +461,6 @@ static __always_inline void handle_tc(bool cgroup_skb, struct __sk_buff *skb, bo event->meta.timestamp = bpf_ktime_get_ns(); event->meta.ifindex = skb->ifindex; - GET_CONFIG() - u64 payload_len = (u64)skb->len; event->meta.packet_size = payload_len; if (g.max_payload_size > 0) { @@ -443,10 +468,18 @@ static __always_inline void handle_tc(bool cgroup_skb, struct __sk_buff *skb, bo } event->meta.payload_len = payload_len; - int event_ret = bpf_perf_event_output(skb, &packet_events, BPF_F_CURRENT_CPU | (payload_len << 32), event, - sizeof(struct packet_event_t)); - if (event_ret != 0) { - // debug_log("[ptcpdump] bpf_perf_event_output exec_events failed: %d\n", event_ret); + if (use_ringbuf) { + if (payload_len > 0) { + struct skb_data_t *skb_data = (struct skb_data_t *)(event + 1); + bpf_skb_load_bytes(skb, 0, &skb_data->data, payload_len); + } + bpf_ringbuf_submit(ringbuf, 0); + } else { + int event_ret = bpf_perf_event_output(skb, &packet_events, BPF_F_CURRENT_CPU | (payload_len << 32), event, + sizeof(struct packet_event_t)); + if (event_ret != 0) { + // debug_log("[ptcpdump] bpf_perf_event_output exec_events failed: %d\n", event_ret); + } } return; diff --git a/go.mod b/go.mod index 89745e94..2b96741a 100644 --- a/go.mod +++ b/go.mod @@ -105,7 +105,7 @@ require ( ) replace ( - github.com/cilium/ebpf => github.com/mozillazg/ebpf v0.17.2-0.20250118135027-62b13edacbea + github.com/cilium/ebpf => github.com/mozillazg/ebpf v0.17.2-0.20250125052608-c310b16eb51c // github.com/cilium/ebpf => ../../cilium/ebpf github.com/gopacket/gopacket => github.com/mozillazg/gopacket v0.0.0-20241026043817-048341de5231 // github.com/gopacket/gopacket => ../../gopacket/gopacket diff --git a/go.sum b/go.sum index 20a33be7..388f798f 100644 --- a/go.sum +++ b/go.sum @@ -183,8 +183,8 @@ github.com/mozillazg/cri-api v0.32.0-alpha.1.0.20241019013855-3dc36f8743df h1:90 github.com/mozillazg/cri-api v0.32.0-alpha.1.0.20241019013855-3dc36f8743df/go.mod h1:ca9lKDUa9PmUGVDSSetDQqgf0dyk/NW8u+MpJo7JaYA= github.com/mozillazg/cri-client v0.31.0-alpha.0.0.20241019023238-87687176fd67 h1:M4+V89TNUGmRgSJZcH2nvotyqnFkmDl+MGApFoZbJY0= github.com/mozillazg/cri-client v0.31.0-alpha.0.0.20241019023238-87687176fd67/go.mod h1:pFm23AAi/gIlW9FGrWPTPnGe1xsyGHPFFO/zezc4w90= -github.com/mozillazg/ebpf v0.17.2-0.20250118135027-62b13edacbea h1:s8OLenj15Oz0UbrFwwJrTUuQD+U0j4glAsybRufNN8s= -github.com/mozillazg/ebpf v0.17.2-0.20250118135027-62b13edacbea/go.mod h1:vay2FaYSmIlv3r8dNACd4mW/OCaZLJKJOo+IHBvCIO8= +github.com/mozillazg/ebpf v0.17.2-0.20250125052608-c310b16eb51c h1:pl6FQF4q0BF8KNYPrnuBuSRqtx8RbXK4lr4gq/lZdRE= +github.com/mozillazg/ebpf v0.17.2-0.20250125052608-c310b16eb51c/go.mod h1:vay2FaYSmIlv3r8dNACd4mW/OCaZLJKJOo+IHBvCIO8= github.com/mozillazg/gopacket v0.0.0-20241026043817-048341de5231 h1:uvhf0oGPfJ24Lc5/N2ysh9sQc71Pog67zxKiLVS4/Qg= github.com/mozillazg/gopacket v0.0.0-20241026043817-048341de5231/go.mod h1:WnFrU1Xkf5lWKV38uKNR9+yYtppn+ZYzOyNqMeH4oNE= github.com/mozillazg/pktdump v0.0.9-0.20241102131745-63c34f34f0d1 h1:kVgiW9P9tfqPXrIU/ZKC8XwkhHkri3s439D+RqSpF6g= diff --git a/internal/consumer/exec.go b/internal/consumer/exec.go index c74f4b5a..d0f18d05 100644 --- a/internal/consumer/exec.go +++ b/internal/consumer/exec.go @@ -2,6 +2,7 @@ package consumer import ( "context" + "github.com/mozillazg/ptcpdump/internal/utils" "sync" "github.com/mozillazg/ptcpdump/bpf" @@ -46,6 +47,7 @@ func (c *ExecEventConsumer) worker(ctx context.Context, ch <-chan bpf.BpfExecEve } func (c *ExecEventConsumer) handleExecEvent(et bpf.BpfExecEventT) { + log.Infof("new exec event: pid: %d, comm: %s", et.Meta.Pid, utils.GoString(et.Filename[:])) e, err := event.ParseProcessExecEvent(et) if err != nil { log.Errorf("[ExecEventConsumer] parse event failed: %s", err) diff --git a/internal/consumer/exit.go b/internal/consumer/exit.go index f4ec66c7..b08e9252 100644 --- a/internal/consumer/exit.go +++ b/internal/consumer/exit.go @@ -2,6 +2,7 @@ package consumer import ( "context" + "github.com/mozillazg/ptcpdump/internal/log" "sync" "github.com/mozillazg/ptcpdump/bpf" @@ -44,6 +45,7 @@ func (c *ExitEventConsumer) worker(ctx context.Context, ch <-chan bpf.BpfExitEve } func (c *ExitEventConsumer) handleExitEvent(et bpf.BpfExitEventT) { + log.Infof("new exit event: pid: %d", et.Pid) c.pcache.MarkDead(int(et.Pid)) } diff --git a/internal/consumer/keylog.go b/internal/consumer/keylog.go index 120239a5..e381038b 100644 --- a/internal/consumer/keylog.go +++ b/internal/consumer/keylog.go @@ -53,6 +53,7 @@ func (c *GoKeyLogEventConsumer) handleEvent(e bpf.BpfGoKeylogEventT) { secret := utils.GoBytes(e.Secret[:int(e.SecretLen)]) line := fmt.Sprintf("%s %x %x\n", label, clientRandom, secret) + log.Infof("new key log event: %s", line) c.write(line) } diff --git a/internal/log/log.go b/internal/log/log.go index 6be52c54..00b5d613 100644 --- a/internal/log/log.go +++ b/internal/log/log.go @@ -23,7 +23,7 @@ var defaultLogger = log.Logger{ var debugLogger = log.Logger{ Level: log.WarnLevel, Caller: 2, - TimeFormat: time.DateTime, + TimeFormat: time.RFC3339Nano, Writer: &log.ConsoleWriter{ Formatter: func(w io.Writer, a *log.FormatterArgs) (int, error) { suffix := "" diff --git a/vendor/github.com/cilium/ebpf/collection.go b/vendor/github.com/cilium/ebpf/collection.go index e0cb9186..34025fec 100644 --- a/vendor/github.com/cilium/ebpf/collection.go +++ b/vendor/github.com/cilium/ebpf/collection.go @@ -26,6 +26,7 @@ type CollectionOptions struct { IgnoreUnknownProgram bool IgnoreNotSupportedProgram bool IgnoreUnknownVariable bool + IgnoreInvalidMap bool // MapReplacements takes a set of Maps that will be used instead of // creating new ones when loading the CollectionSpec. @@ -285,7 +286,13 @@ func (cs *CollectionSpec) LoadAndAssign(to interface{}, opts *CollectionOptions) case reflect.TypeOf((*Map)(nil)): assignedMaps[name] = true - return loader.loadMap(name) + m, err := loader.loadMap(name) + if err != nil { + if strings.Contains(err.Error(), "invalid argument") && opts.IgnoreInvalidMap { + return nil, nil + } + } + return m, err case reflect.TypeOf((*Variable)(nil)): assignedVars[name] = true diff --git a/vendor/github.com/cilium/ebpf/ringbuf/doc.go b/vendor/github.com/cilium/ebpf/ringbuf/doc.go new file mode 100644 index 00000000..9e450121 --- /dev/null +++ b/vendor/github.com/cilium/ebpf/ringbuf/doc.go @@ -0,0 +1,6 @@ +// Package ringbuf allows interacting with Linux BPF ring buffer. +// +// BPF allows submitting custom events to a BPF ring buffer map set up +// by userspace. This is very useful to push things like packet samples +// from BPF to a daemon running in user space. +package ringbuf diff --git a/vendor/github.com/cilium/ebpf/ringbuf/reader.go b/vendor/github.com/cilium/ebpf/ringbuf/reader.go new file mode 100644 index 00000000..11ef93b4 --- /dev/null +++ b/vendor/github.com/cilium/ebpf/ringbuf/reader.go @@ -0,0 +1,208 @@ +package ringbuf + +import ( + "errors" + "fmt" + "os" + "sync" + "time" + "unsafe" + + "github.com/cilium/ebpf" + "github.com/cilium/ebpf/internal/epoll" + "github.com/cilium/ebpf/internal/sys" + "github.com/cilium/ebpf/internal/unix" +) + +var ( + ErrClosed = os.ErrClosed + ErrFlushed = epoll.ErrFlushed + errEOR = errors.New("end of ring") + errBusy = errors.New("sample not committed yet") +) + +// ringbufHeader from 'struct bpf_ringbuf_hdr' in kernel/bpf/ringbuf.c +type ringbufHeader struct { + Len uint32 + _ uint32 // pg_off, only used by kernel internals +} + +const ringbufHeaderSize = int(unsafe.Sizeof(ringbufHeader{})) + +func (rh *ringbufHeader) isBusy() bool { + return rh.Len&sys.BPF_RINGBUF_BUSY_BIT != 0 +} + +func (rh *ringbufHeader) isDiscard() bool { + return rh.Len&sys.BPF_RINGBUF_DISCARD_BIT != 0 +} + +func (rh *ringbufHeader) dataLen() int { + return int(rh.Len & ^uint32(sys.BPF_RINGBUF_BUSY_BIT|sys.BPF_RINGBUF_DISCARD_BIT)) +} + +type Record struct { + RawSample []byte + + // The minimum number of bytes remaining in the ring buffer after this Record has been read. + Remaining int +} + +// Reader allows reading bpf_ringbuf_output +// from user space. +type Reader struct { + poller *epoll.Poller + + // mu protects read/write access to the Reader structure + mu sync.Mutex + ring *ringbufEventRing + epollEvents []unix.EpollEvent + haveData bool + deadline time.Time + bufferSize int + pendingErr error +} + +// NewReader creates a new BPF ringbuf reader. +func NewReader(ringbufMap *ebpf.Map) (*Reader, error) { + if ringbufMap.Type() != ebpf.RingBuf { + return nil, fmt.Errorf("invalid Map type: %s", ringbufMap.Type()) + } + + maxEntries := int(ringbufMap.MaxEntries()) + if maxEntries == 0 || (maxEntries&(maxEntries-1)) != 0 { + return nil, fmt.Errorf("ringbuffer map size %d is zero or not a power of two", maxEntries) + } + + poller, err := epoll.New() + if err != nil { + return nil, err + } + + if err := poller.Add(ringbufMap.FD(), 0); err != nil { + poller.Close() + return nil, err + } + + ring, err := newRingBufEventRing(ringbufMap.FD(), maxEntries) + if err != nil { + poller.Close() + return nil, fmt.Errorf("failed to create ringbuf ring: %w", err) + } + + return &Reader{ + poller: poller, + ring: ring, + epollEvents: make([]unix.EpollEvent, 1), + bufferSize: ring.size(), + }, nil +} + +// Close frees resources used by the reader. +// +// It interrupts calls to Read. +func (r *Reader) Close() error { + if err := r.poller.Close(); err != nil { + if errors.Is(err, os.ErrClosed) { + return nil + } + return err + } + + // Acquire the lock. This ensures that Read isn't running. + r.mu.Lock() + defer r.mu.Unlock() + + if r.ring != nil { + r.ring.Close() + r.ring = nil + } + + return nil +} + +// SetDeadline controls how long Read and ReadInto will block waiting for samples. +// +// Passing a zero time.Time will remove the deadline. +func (r *Reader) SetDeadline(t time.Time) { + r.mu.Lock() + defer r.mu.Unlock() + + r.deadline = t +} + +// Read the next record from the BPF ringbuf. +// +// Calling [Close] interrupts the method with [os.ErrClosed]. Calling [Flush] +// makes it return all records currently in the ring buffer, followed by [ErrFlushed]. +// +// Returns [os.ErrDeadlineExceeded] if a deadline was set and after all records +// have been read from the ring. +// +// See [ReadInto] for a more efficient version of this method. +func (r *Reader) Read() (Record, error) { + var rec Record + return rec, r.ReadInto(&rec) +} + +// ReadInto is like Read except that it allows reusing Record and associated buffers. +func (r *Reader) ReadInto(rec *Record) error { + r.mu.Lock() + defer r.mu.Unlock() + + if r.ring == nil { + return fmt.Errorf("ringbuffer: %w", ErrClosed) + } + + for { + if !r.haveData { + if pe := r.pendingErr; pe != nil { + r.pendingErr = nil + return pe + } + + _, err := r.poller.Wait(r.epollEvents[:cap(r.epollEvents)], r.deadline) + if errors.Is(err, os.ErrDeadlineExceeded) || errors.Is(err, ErrFlushed) { + // Ignoring this for reading a valid entry after timeout or flush. + // This can occur if the producer submitted to the ring buffer + // with BPF_RB_NO_WAKEUP. + r.pendingErr = err + } else if err != nil { + return err + } + r.haveData = true + } + + for { + err := r.ring.readRecord(rec) + // Not using errors.Is which is quite a bit slower + // For a tight loop it might make a difference + if err == errBusy { + continue + } + if err == errEOR { + r.haveData = false + break + } + return err + } + } +} + +// BufferSize returns the size in bytes of the ring buffer +func (r *Reader) BufferSize() int { + return r.bufferSize +} + +// Flush unblocks Read/ReadInto and successive Read/ReadInto calls will return pending samples at this point, +// until you receive a ErrFlushed error. +func (r *Reader) Flush() error { + return r.poller.Flush() +} + +// AvailableBytes returns the amount of data available to read in the ring buffer in bytes. +func (r *Reader) AvailableBytes() int { + // Don't need to acquire the lock here since the implementation of AvailableBytes + // performs atomic loads on the producer and consumer positions. + return int(r.ring.AvailableBytes()) +} diff --git a/vendor/github.com/cilium/ebpf/ringbuf/ring.go b/vendor/github.com/cilium/ebpf/ringbuf/ring.go new file mode 100644 index 00000000..8f75bba8 --- /dev/null +++ b/vendor/github.com/cilium/ebpf/ringbuf/ring.go @@ -0,0 +1,145 @@ +package ringbuf + +import ( + "fmt" + "io" + "os" + "runtime" + "sync/atomic" + "unsafe" + + "github.com/cilium/ebpf/internal" + "github.com/cilium/ebpf/internal/sys" + "github.com/cilium/ebpf/internal/unix" +) + +type ringbufEventRing struct { + prod []byte + cons []byte + *ringReader +} + +func newRingBufEventRing(mapFD, size int) (*ringbufEventRing, error) { + cons, err := unix.Mmap(mapFD, 0, os.Getpagesize(), unix.PROT_READ|unix.PROT_WRITE, unix.MAP_SHARED) + if err != nil { + return nil, fmt.Errorf("can't mmap consumer page: %w", err) + } + + prod, err := unix.Mmap(mapFD, (int64)(os.Getpagesize()), os.Getpagesize()+2*size, unix.PROT_READ, unix.MAP_SHARED) + if err != nil { + _ = unix.Munmap(cons) + return nil, fmt.Errorf("can't mmap data pages: %w", err) + } + + cons_pos := (*uint64)(unsafe.Pointer(&cons[0])) + prod_pos := (*uint64)(unsafe.Pointer(&prod[0])) + + ring := &ringbufEventRing{ + prod: prod, + cons: cons, + ringReader: newRingReader(cons_pos, prod_pos, prod[os.Getpagesize():]), + } + runtime.SetFinalizer(ring, (*ringbufEventRing).Close) + + return ring, nil +} + +func (ring *ringbufEventRing) Close() { + runtime.SetFinalizer(ring, nil) + + _ = unix.Munmap(ring.prod) + _ = unix.Munmap(ring.cons) + + ring.prod = nil + ring.cons = nil +} + +type ringReader struct { + // These point into mmap'ed memory and must be accessed atomically. + prod_pos, cons_pos *uint64 + mask uint64 + ring []byte +} + +func newRingReader(cons_ptr, prod_ptr *uint64, ring []byte) *ringReader { + return &ringReader{ + prod_pos: prod_ptr, + cons_pos: cons_ptr, + // cap is always a power of two + mask: uint64(cap(ring)/2 - 1), + ring: ring, + } +} + +// To be able to wrap around data, data pages in ring buffers are mapped twice in +// a single contiguous virtual region. +// Therefore the returned usable size is half the size of the mmaped region. +func (rr *ringReader) size() int { + return cap(rr.ring) / 2 +} + +// The amount of data available to read in the ring buffer. +func (rr *ringReader) AvailableBytes() uint64 { + prod := atomic.LoadUint64(rr.prod_pos) + cons := atomic.LoadUint64(rr.cons_pos) + return prod - cons +} + +// Read a record from an event ring. +func (rr *ringReader) readRecord(rec *Record) error { + prod := atomic.LoadUint64(rr.prod_pos) + cons := atomic.LoadUint64(rr.cons_pos) + + for { + if remaining := prod - cons; remaining == 0 { + return errEOR + } else if remaining < sys.BPF_RINGBUF_HDR_SZ { + return fmt.Errorf("read record header: %w", io.ErrUnexpectedEOF) + } + + // read the len field of the header atomically to ensure a happens before + // relationship with the xchg in the kernel. Without this we may see len + // without BPF_RINGBUF_BUSY_BIT before the written data is visible. + // See https://github.com/torvalds/linux/blob/v6.8/kernel/bpf/ringbuf.c#L484 + start := cons & rr.mask + len := atomic.LoadUint32((*uint32)((unsafe.Pointer)(&rr.ring[start]))) + header := ringbufHeader{Len: len} + + if header.isBusy() { + // the next sample in the ring is not committed yet so we + // exit without storing the reader/consumer position + // and start again from the same position. + return errBusy + } + + cons += sys.BPF_RINGBUF_HDR_SZ + + // Data is always padded to 8 byte alignment. + dataLenAligned := uint64(internal.Align(header.dataLen(), 8)) + if remaining := prod - cons; remaining < dataLenAligned { + return fmt.Errorf("read sample data: %w", io.ErrUnexpectedEOF) + } + + start = cons & rr.mask + cons += dataLenAligned + + if header.isDiscard() { + // when the record header indicates that the data should be + // discarded, we skip it by just updating the consumer position + // to the next record. + atomic.StoreUint64(rr.cons_pos, cons) + continue + } + + if n := header.dataLen(); cap(rec.RawSample) < n { + rec.RawSample = make([]byte, n) + } else { + rec.RawSample = rec.RawSample[:n] + } + + copy(rec.RawSample, rr.ring[start:]) + rec.Remaining = int(prod - cons) + atomic.StoreUint64(rr.cons_pos, cons) + return nil + } +} diff --git a/vendor/modules.txt b/vendor/modules.txt index f0d4f1b6..eb52540f 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -42,7 +42,7 @@ github.com/Microsoft/hcsshim/internal/wclayer github.com/Microsoft/hcsshim/internal/winapi github.com/Microsoft/hcsshim/osversion github.com/Microsoft/hcsshim/pkg/ociwclayer -# github.com/cilium/ebpf v0.16.0 => github.com/mozillazg/ebpf v0.17.2-0.20250118135027-62b13edacbea +# github.com/cilium/ebpf v0.16.0 => github.com/mozillazg/ebpf v0.17.2-0.20250125052608-c310b16eb51c ## explicit; go 1.22 github.com/cilium/ebpf github.com/cilium/ebpf/asm @@ -63,6 +63,7 @@ github.com/cilium/ebpf/internal/tracefs github.com/cilium/ebpf/internal/unix github.com/cilium/ebpf/link github.com/cilium/ebpf/perf +github.com/cilium/ebpf/ringbuf github.com/cilium/ebpf/rlimit # github.com/cloudflare/cbpfc v0.0.0-20230809125630-31aa294050ff ## explicit; go 1.19 @@ -601,7 +602,7 @@ k8s.io/utils/exec ## explicit; go 1.12 rsc.io/binaryregexp rsc.io/binaryregexp/syntax -# github.com/cilium/ebpf => github.com/mozillazg/ebpf v0.17.2-0.20250118135027-62b13edacbea +# github.com/cilium/ebpf => github.com/mozillazg/ebpf v0.17.2-0.20250125052608-c310b16eb51c # github.com/gopacket/gopacket => github.com/mozillazg/gopacket v0.0.0-20241026043817-048341de5231 # github.com/x-way/pktdump => github.com/mozillazg/pktdump v0.0.9-0.20241102131745-63c34f34f0d1 # k8s.io/cri-api => github.com/mozillazg/cri-api v0.32.0-alpha.1.0.20241019013855-3dc36f8743df