diff --git a/kern/common.h b/kern/common.h index 0e6bdfbab..d42f31f27 100644 --- a/kern/common.h +++ b/kern/common.h @@ -39,7 +39,6 @@ #define AF_INET 2 #define AF_INET6 10 -#define SA_DATA_LEN 14 #define BASH_ERRNO_DEFAULT 128 #define BASH_EVENT_TYPE_READLINE 0 diff --git a/kern/openssl.h b/kern/openssl.h index 36ec90435..e0108ff43 100644 --- a/kern/openssl.h +++ b/kern/openssl.h @@ -43,9 +43,12 @@ struct connect_event_t { u32 pid; u32 tid; u32 fd; - char sa_data[SA_DATA_LEN]; + u16 sport; + u16 dport; + __be32 saddr; + __be32 daddr; char comm[TASK_COMM_LEN]; -}; +} __attribute__((packed)); // NOTE: do not leave padding hole in this struct. struct active_ssl_buf { /* @@ -59,6 +62,11 @@ struct active_ssl_buf { const char* buf; }; +struct tcp_fd_info { + u64 file; + int fd; +}; + /*********************************************************** * BPF MAPS ***********************************************************/ @@ -114,6 +122,14 @@ struct { } ssl_st_fd SEC(".maps"); +struct { + __uint(type, BPF_MAP_TYPE_HASH); + __type(key, u64); + __type(value, struct tcp_fd_info); + __uint(max_entries, 10240); +} tcp_fd_infos SEC(".maps"); + + /*********************************************************** * General helper functions ***********************************************************/ @@ -436,14 +452,67 @@ int probe_ret_SSL_read(struct pt_regs* ctx) { return 0; } + +static __inline struct tcp_fd_info *find_fd_info(struct pt_regs *regs) { + u64 pid_tgid = bpf_get_current_pid_tgid(); + + return bpf_map_lookup_elem(&tcp_fd_infos, &pid_tgid); +} + +static __inline struct tcp_fd_info *lookup_and_delete_fd_info(struct pt_regs *regs) { + struct tcp_fd_info *fd_info; + u64 pid_tgid; + + pid_tgid = bpf_get_current_pid_tgid(); + fd_info = bpf_map_lookup_elem(&tcp_fd_infos, &pid_tgid); + if (fd_info) { + bpf_map_delete_elem(&tcp_fd_infos, &pid_tgid); + } + return fd_info; +} + +static __inline struct sock *tcp_sock_from_file(u64 ptr) { + struct socket *socket; + struct file *file; + struct sock *sk; + + file = (struct file *)ptr; + bpf_probe_read_kernel(&socket, sizeof(socket), &file->private_data); + bpf_probe_read_kernel(&sk, sizeof(sk), &socket->sk); + return sk; +} + // libc : int __connect (int fd, __CONST_SOCKADDR_ARG addr, socklen_t len) // kernel : int __sys_connect(int fd, struct sockaddr __user *uservaddr, int addrlen) SEC("kprobe/sys_connect") int probe_connect(struct pt_regs* ctx) { + u64 pid_tgid = bpf_get_current_pid_tgid(); + struct tcp_fd_info fd_info = {}; + + fd_info.fd = PT_REGS_PARM1(ctx); + bpf_map_update_elem(&tcp_fd_infos, &pid_tgid, &fd_info, BPF_ANY); + return 0; +} + +SEC("kprobe/__sys_connect_file") +int probe_connect_file(struct pt_regs* ctx) { + struct tcp_fd_info *fd_info; + + fd_info = find_fd_info(ctx); + if (fd_info) { + fd_info->file = (u64)(void *) PT_REGS_PARM1(ctx); + } + return 0; +} + +static __inline int kretprobe_connect(struct pt_regs *ctx, int fd, struct sock *sk, const bool active) { u64 current_pid_tgid = bpf_get_current_pid_tgid(); u32 pid = current_pid_tgid >> 32; u64 current_uid_gid = bpf_get_current_uid_gid(); u32 uid = current_uid_gid; + u16 address_family = 0; + u64 addrs; + u32 ports; #ifndef KERNEL_LESS_5_2 // if target_ppid is 0 then we target all pids @@ -455,16 +524,15 @@ int probe_connect(struct pt_regs* ctx) { } #endif - u32 fd = (u32)PT_REGS_PARM1(ctx); - struct sockaddr* saddr = (struct sockaddr*)PT_REGS_PARM2(ctx); - if (!saddr) { + bpf_probe_read_kernel(&address_family, sizeof(address_family), &sk->__sk_common.skc_family); + if (address_family != AF_INET) { return 0; } - sa_family_t address_family = 0; - bpf_probe_read_user(&address_family, sizeof(address_family), - &saddr->sa_family); - if (address_family != AF_INET) { + // if the connection hasn't been established yet, the ports or addrs are 0. + bpf_probe_read_kernel(&addrs, sizeof(addrs), &sk->__sk_common.skc_addrpair); + bpf_probe_read_kernel(&ports, sizeof(ports), &sk->__sk_common.skc_portpair); + if (ports == 0 || addrs == 0) { return 0; } @@ -476,7 +544,17 @@ int probe_connect(struct pt_regs* ctx) { conn.pid = pid; conn.tid = current_pid_tgid; conn.fd = fd; - bpf_probe_read_user(&conn.sa_data, SA_DATA_LEN, &saddr->sa_data); + if (active) { + conn.dport = bpf_ntohs((u16)ports); + conn.sport = ports >> 16; + conn.daddr = (__be32)addrs; + conn.saddr = (__be32)(addrs >> 32); + } else { + conn.sport = bpf_ntohs((u16)ports); + conn.dport = ports >> 16; + conn.saddr = (__be32)addrs; + conn.daddr = (__be32)(addrs >> 32); + } bpf_get_current_comm(&conn.comm, sizeof(conn.comm)); bpf_perf_event_output(ctx, &connect_events, BPF_F_CURRENT_CPU, &conn, @@ -484,6 +562,63 @@ int probe_connect(struct pt_regs* ctx) { return 0; } +SEC("kretprobe/sys_connect") +int retprobe_connect(struct pt_regs* ctx) { + struct tcp_fd_info *fd_info; + struct sock *sk; + + fd_info = lookup_and_delete_fd_info(ctx); + if (fd_info) { + sk = tcp_sock_from_file(fd_info->file); + if (sk) { + return kretprobe_connect(ctx, fd_info->fd, sk, true); + } + } + return 0; +} + +#ifndef IS_ERR_VALUE +#define MAX_ERRNO 4095 +#define IS_ERR_VALUE(x) ((unsigned long)(void *)(x) >= (unsigned long)-MAX_ERRNO) +#endif + +SEC("kretprobe/do_accept") +int retprobe_do_accept(struct pt_regs* ctx) { + struct tcp_fd_info *fd_info; + struct file *file; + + file = (struct file *)PT_REGS_RC(ctx); + if (IS_ERR_VALUE(file)) { + return 0; + } + + fd_info = find_fd_info(ctx); + if (fd_info) { + fd_info->file = (u64)file; + } + return 0; +} + +SEC("kretprobe/__sys_accept4") +int retprobe_accept4(struct pt_regs* ctx) { + struct tcp_fd_info *fd_info; + struct sock *sk; + int fd; + + fd = PT_REGS_RC(ctx); + if (fd < 0) { + return 0; + } + + fd_info = lookup_and_delete_fd_info(ctx); + if (fd_info) { + sk = tcp_sock_from_file(fd_info->file); + if (sk) { + return kretprobe_connect(ctx, fd, sk, false); + } + } + return 0; +} // int SSL_set_fd(SSL *s, int fd) diff --git a/pkg/event_processor/base_event.go b/pkg/event_processor/base_event.go index abda81ca6..297ca983e 100644 --- a/pkg/event_processor/base_event.go +++ b/pkg/event_processor/base_event.go @@ -18,6 +18,7 @@ import ( "bytes" "encoding/binary" "fmt" + "github.com/gojue/ecapture/user/event" ) @@ -35,8 +36,6 @@ const ChunkSizeHalf = ChunkSize / 2 const MaxDataSize = 1024 * 4 -//const SaDataLen = 14 - const ( Ssl2Version = 0x0002 Ssl3Version = 0x0300 diff --git a/user/event/event_openssl.go b/user/event/event_openssl.go index 9e80cda6c..b587c8b32 100644 --- a/user/event/event_openssl.go +++ b/user/event/event_openssl.go @@ -18,7 +18,9 @@ import ( "bytes" "encoding/binary" "fmt" - "net" + "net/netip" + "strings" + "unsafe" ) type AttachType int64 @@ -29,7 +31,6 @@ const ( ) const MaxDataSize = 1024 * 4 -const SaDataLen = 14 const ( Ssl2Version = 0x0002 @@ -79,7 +80,7 @@ type SSLDataEvent struct { Comm [16]byte `json:"Comm"` Fd uint32 `json:"fd"` Version int32 `json:"version"` - Addr string + Tuple string BioType uint32 } @@ -123,8 +124,12 @@ func (se *SSLDataEvent) Decode(payload []byte) (err error) { return nil } +func commStr(comm []byte) string { + return strings.TrimSpace(CToGoString(comm)) +} + func (se *SSLDataEvent) GetUUID() string { - return fmt.Sprintf("%d_%d_%s_%d_%d_%s", se.Pid, se.Tid, CToGoString(se.Comm[:]), se.Fd, se.DataType, se.Addr) + return fmt.Sprintf("%d_%d_%s_%d_%d_%s", se.Pid, se.Tid, commStr(se.Comm[:]), se.Fd, se.DataType, se.Tuple) } func (se *SSLDataEvent) Payload() []byte { @@ -138,45 +143,45 @@ func (se *SSLDataEvent) PayloadLen() int { func (se *SSLDataEvent) StringHex() string { //addr := se.module.(*module.MOpenSSLProbe).GetConn(se.Pid, se.Fd) addr := "[TODO]" - var perfix, connInfo string + var prefix, connInfo string switch AttachType(se.DataType) { case ProbeEntry: connInfo = fmt.Sprintf("%sRecived %d%s bytes from %s%s%s", COLORGREEN, se.DataLen, COLORRESET, COLORYELLOW, addr, COLORRESET) - perfix = COLORGREEN + prefix = COLORGREEN case ProbeRet: connInfo = fmt.Sprintf("%sSend %d%s bytes to %s%s%s", COLORPURPLE, se.DataLen, COLORRESET, COLORYELLOW, addr, COLORRESET) - perfix = fmt.Sprintf("%s\t", COLORPURPLE) + prefix = fmt.Sprintf("%s\t", COLORPURPLE) default: - perfix = fmt.Sprintf("UNKNOW_%d", se.DataType) + prefix = fmt.Sprintf("UNKNOW_%d", se.DataType) } - b := dumpByteSlice(se.Data[:se.DataLen], perfix) + b := dumpByteSlice(se.Data[:se.DataLen], prefix) b.WriteString(COLORRESET) v := TlsVersion{Version: se.Version} - s := fmt.Sprintf("PID:%d, Comm:%s, TID:%d, %s, Version:%s, Payload:\n%s", se.Pid, CToGoString(se.Comm[:]), se.Tid, connInfo, v.String(), b.String()) + s := fmt.Sprintf("PID:%d, Comm:%s, TID:%d, %s, Version:%s, Payload:\n%s", se.Pid, commStr(se.Comm[:]), se.Tid, connInfo, v.String(), b.String()) return s } func (se *SSLDataEvent) String() string { //addr := se.module.(*module.MOpenSSLProbe).GetConn(se.Pid, se.Fd) addr := "[TODO]" - if se.Addr != "" { - addr = se.Addr + if se.Tuple != "" { + addr = se.Tuple } - var perfix, connInfo string + var prefix, connInfo string switch AttachType(se.DataType) { case ProbeEntry: connInfo = fmt.Sprintf("%sRecived %d%s bytes from %s%s%s", COLORGREEN, se.DataLen, COLORRESET, COLORYELLOW, addr, COLORRESET) - perfix = COLORGREEN + prefix = COLORGREEN case ProbeRet: connInfo = fmt.Sprintf("%sSend %d%s bytes to %s%s%s", COLORPURPLE, se.DataLen, COLORRESET, COLORYELLOW, addr, COLORRESET) - perfix = COLORPURPLE + prefix = COLORPURPLE default: connInfo = fmt.Sprintf("%sUNKNOW_%d%s", COLORRED, se.DataType, COLORRESET) } v := TlsVersion{Version: se.Version} - s := fmt.Sprintf("PID:%d, Comm:%s, TID:%d, Version:%s, %s, Payload:\n%s%s%s", se.Pid, bytes.TrimSpace(se.Comm[:]), se.Tid, v.String(), connInfo, perfix, string(se.Data[:se.DataLen]), COLORRESET) + s := fmt.Sprintf("PID:%d, Comm:%s, TID:%d, Version:%s, %s, Payload:\n%s%s%s", se.Pid, commStr(se.Comm[:]), se.Tid, v.String(), connInfo, prefix, string(se.Data[:se.DataLen]), COLORRESET) return s } @@ -192,57 +197,49 @@ func (se *SSLDataEvent) EventType() EventType { // connect_events map /* -uint64_t timestamp_ns; + uint64_t timestamp_ns; uint32_t pid; uint32_t tid; uint32_t fd; - char sa_data[SA_DATA_LEN]; + char ports[4]; + char addrs[8]; char Comm[TASK_COMM_LEN]; */ +type connDataEvent struct { + TimestampNs uint64 `json:"timestampNs"` + Pid uint32 `json:"pid"` + Tid uint32 `json:"tid"` + Fd uint32 `json:"fd"` + Sport uint16 `json:"sport"` + Dport uint16 `json:"dport"` + Saddr [4]byte `json:"saddr"` + Daddr [4]byte `json:"daddr"` + Comm [16]byte `json:"Comm"` + + // NOTE: do not leave padding hole in this struct. +} type ConnDataEvent struct { - eventType EventType - TimestampNs uint64 `json:"timestampNs"` - Pid uint32 `json:"pid"` - Tid uint32 `json:"tid"` - Fd uint32 `json:"fd"` - SaData [SaDataLen]byte `json:"saData"` - Comm [16]byte `json:"Comm"` - Addr string `json:"addr"` + eventType EventType + connDataEvent + Tuple string `json:"tuple"` } func (ce *ConnDataEvent) Decode(payload []byte) (err error) { - buf := bytes.NewBuffer(payload) - if err = binary.Read(buf, binary.LittleEndian, &ce.TimestampNs); err != nil { - return - } - if err = binary.Read(buf, binary.LittleEndian, &ce.Pid); err != nil { - return - } - if err = binary.Read(buf, binary.LittleEndian, &ce.Tid); err != nil { - return - } - if err = binary.Read(buf, binary.LittleEndian, &ce.Fd); err != nil { - return - } - if err = binary.Read(buf, binary.LittleEndian, &ce.SaData); err != nil { - return - } - if err = binary.Read(buf, binary.LittleEndian, &ce.Comm); err != nil { - return - } - port := binary.BigEndian.Uint16(ce.SaData[0:2]) - ip := net.IPv4(ce.SaData[2], ce.SaData[3], ce.SaData[4], ce.SaData[5]) - ce.Addr = fmt.Sprintf("%s:%d", ip, port) + data := unsafe.Slice((*byte)(unsafe.Pointer(&ce.connDataEvent)), int(unsafe.Sizeof(ce.connDataEvent))) + copy(data, payload) + + saddr, daddr := netip.AddrFrom4(ce.Saddr), netip.AddrFrom4(ce.Daddr) + ce.Tuple = fmt.Sprintf("%s:%d-%s:%d", saddr, ce.Sport, daddr, ce.Dport) return nil } func (ce *ConnDataEvent) StringHex() string { - s := fmt.Sprintf("PID:%d, Comm:%s, TID:%d, FD:%d, Addr: %s", ce.Pid, bytes.TrimSpace(ce.Comm[:]), ce.Tid, ce.Fd, ce.Addr) + s := fmt.Sprintf("PID:%d, Comm:%s, TID:%d, FD:%d, Tuple: %s", ce.Pid, commStr(ce.Comm[:]), ce.Tid, ce.Fd, ce.Tuple) return s } func (ce *ConnDataEvent) String() string { - s := fmt.Sprintf("PID:%d, Comm:%s, TID:%d, FD:%d, Addr: %s", ce.Pid, bytes.TrimSpace(ce.Comm[:]), ce.Tid, ce.Fd, ce.Addr) + s := fmt.Sprintf("PID:%d, Comm:%s, TID:%d, FD:%d, Tuple: %s", ce.Pid, commStr(ce.Comm[:]), ce.Tid, ce.Fd, ce.Tuple) return s } @@ -257,13 +254,13 @@ func (ce *ConnDataEvent) EventType() EventType { } func (ce *ConnDataEvent) GetUUID() string { - return fmt.Sprintf("%d_%d_%s_%d", ce.Pid, ce.Tid, bytes.TrimSpace(ce.Comm[:]), ce.Fd) + return fmt.Sprintf("%d_%d_%s_%d", ce.Pid, ce.Tid, commStr(ce.Comm[:]), ce.Fd) } func (ce *ConnDataEvent) Payload() []byte { - return []byte(ce.Addr) + return []byte(ce.Tuple) } func (ce *ConnDataEvent) PayloadLen() int { - return len(ce.Addr) + return len(ce.Tuple) } diff --git a/user/module/probe_openssl.go b/user/module/probe_openssl.go index 2bc07dad9..12bbfcc86 100644 --- a/user/module/probe_openssl.go +++ b/user/module/probe_openssl.go @@ -20,7 +20,6 @@ import ( "crypto" "errors" "fmt" - "github.com/rs/zerolog" "io" "os" "path/filepath" @@ -35,12 +34,13 @@ import ( "github.com/cilium/ebpf" manager "github.com/gojue/ebpfmanager" + "github.com/rs/zerolog" "golang.org/x/sys/unix" ) const ( - ConnNotFound = "[ADDR_NOT_FOUND]" - DefaultAddr = "0.0.0.0" + ConnNotFound = "[TUPLE_NOT_FOUND]" + DefaultTuple = "0.0.0.0:0-0.0.0.0:0" // OpenSSL the classes of BIOs // https://github.com/openssl/openssl/blob/openssl-3.0.0/include/openssl/bio.h.in BioTypeDescriptor = 0x0100 @@ -393,9 +393,9 @@ func (m *MOpenSSLProbe) Events() []*ebpf.Map { return m.eventMaps } -func (m *MOpenSSLProbe) AddConn(pid, fd uint32, addr string) { +func (m *MOpenSSLProbe) AddConn(pid, fd uint32, tuple string) { if fd <= 0 { - m.logger.Info().Uint32("pid", pid).Uint32("fd", fd).Str("address", addr).Msg("AddConn failed") + m.logger.Info().Uint32("pid", pid).Uint32("fd", fd).Str("tuple", tuple).Msg("AddConn failed") return } // save @@ -407,9 +407,9 @@ func (m *MOpenSSLProbe) AddConn(pid, fd uint32, addr string) { if !f { connMap = make(map[uint32]string) } - connMap[fd] = addr + connMap[fd] = tuple m.pidConns[pid] = connMap - m.logger.Debug().Uint32("pid", pid).Uint32("fd", fd).Str("address", addr).Msg("AddConn success") + m.logger.Debug().Uint32("pid", pid).Uint32("fd", fd).Str("tuple", tuple).Msg("AddConn success") return } @@ -441,7 +441,7 @@ func (m *MOpenSSLProbe) GetConn(pid, fd uint32) string { if fd <= 0 { return ConnNotFound } - addr := "" + tuple := "" var connMap map[uint32]string var f bool m.logger.Debug().Uint32("pid", pid).Uint32("fd", fd).Msg("GetConn") @@ -451,11 +451,11 @@ func (m *MOpenSSLProbe) GetConn(pid, fd uint32) string { if !f { return ConnNotFound } - addr, f = connMap[fd] + tuple, f = connMap[fd] if !f { return ConnNotFound } - return addr + return tuple } func (m *MOpenSSLProbe) saveMasterSecret(secretEvent *event.MasterSecretEvent) { @@ -708,7 +708,7 @@ func (m *MOpenSSLProbe) Dispatcher(eventStruct event.IEventStruct) { // detect eventStruct type switch eventStruct.(type) { case *event.ConnDataEvent: - m.AddConn(eventStruct.(*event.ConnDataEvent).Pid, eventStruct.(*event.ConnDataEvent).Fd, eventStruct.(*event.ConnDataEvent).Addr) + m.AddConn(eventStruct.(*event.ConnDataEvent).Pid, eventStruct.(*event.ConnDataEvent).Fd, eventStruct.(*event.ConnDataEvent).Tuple) case *event.MasterSecretEvent: m.saveMasterSecret(eventStruct.(*event.MasterSecretEvent)) case *event.MasterSecretBSSLEvent: @@ -726,15 +726,15 @@ func (m *MOpenSSLProbe) Dispatcher(eventStruct event.IEventStruct) { func (m *MOpenSSLProbe) dumpSslData(eventStruct *event.SSLDataEvent) { // BIO_TYPE_SOURCE_SINK|BIO_TYPE_DESCRIPTOR = 0x0400|0x0100 = 1280 if eventStruct.Fd <= 0 && eventStruct.BioType > BioTypeSourceSink|BioTypeDescriptor { - m.logger.Error().Uint32("pid", eventStruct.Pid).Uint32("fd", eventStruct.Fd).Str("address", eventStruct.Addr).Msg("SSLDataEvent's fd is 0") + m.logger.Error().Uint32("pid", eventStruct.Pid).Uint32("fd", eventStruct.Fd).Str("tuple", eventStruct.Tuple).Msg("SSLDataEvent's fd is 0") //return } - addr := m.GetConn(eventStruct.Pid, eventStruct.Fd) - m.logger.Debug().Uint32("pid", eventStruct.Pid).Uint32("bio_type", eventStruct.BioType).Uint32("fd", eventStruct.Fd).Str("address", addr).Msg("SSLDataEvent") - if addr == ConnNotFound { - eventStruct.Addr = DefaultAddr + tuple := m.GetConn(eventStruct.Pid, eventStruct.Fd) + m.logger.Debug().Uint32("pid", eventStruct.Pid).Uint32("bio_type", eventStruct.BioType).Uint32("fd", eventStruct.Fd).Str("tuple", tuple).Msg("SSLDataEvent") + if tuple == ConnNotFound { + eventStruct.Tuple = DefaultTuple } else { - eventStruct.Addr = addr + eventStruct.Tuple = tuple } // m.processor.PcapFile(eventStruct) //if m.conf.GetHex() { diff --git a/user/module/probe_openssl_text.go b/user/module/probe_openssl_text.go index f13ee6e8f..47e8e38f0 100644 --- a/user/module/probe_openssl_text.go +++ b/user/module/probe_openssl_text.go @@ -2,15 +2,16 @@ package module import ( "errors" + "math" + "os" + "path" + "strings" + "github.com/cilium/ebpf" manager "github.com/gojue/ebpfmanager" "github.com/gojue/ecapture/user/config" "github.com/gojue/ecapture/user/event" "golang.org/x/sys/unix" - "math" - "os" - "path" - "strings" ) func (m *MOpenSSLProbe) setupManagersText() error { @@ -76,12 +77,36 @@ func (m *MOpenSSLProbe) setupManagersText() error { AttachToFuncName: "__sys_connect", UID: "kprobe_sys_connect", }, + { + Section: "kprobe/__sys_connect_file", + EbpfFuncName: "probe_connect_file", + AttachToFuncName: "__sys_connect_file", + UID: "kprobe_sys_connect_file", + }, + { + Section: "kretprobe/sys_connect", + EbpfFuncName: "retprobe_connect", + AttachToFuncName: "__sys_connect", + UID: "kretprobe_sys_connect", + }, { Section: "kprobe/sys_connect", EbpfFuncName: "probe_connect", AttachToFuncName: "__sys_accept4", UID: "kprobe_sys_accept4", }, + { + Section: "kretprobe/do_accept", + EbpfFuncName: "retprobe_do_accept", + AttachToFuncName: "do_accept", + UID: "kretprobe_do_accept", + }, + { + Section: "kretprobe/__sys_accept4", + EbpfFuncName: "retprobe_accept4", + AttachToFuncName: "__sys_accept4", + UID: "kretprobe_sys_accept4", + }, // --------------------------------------------------