fix : OpenSSL's file descriptor is always 0

kern/boringssl_1_1_1_kern.c

@@ -10,6 +10,12 @@
 // ssl_st->session
 #define SSL_ST_SESSION 0x58
 
+// ssl_st->rbio
+#define SSL_ST_RBIO 0x18
+
+// ssl_st->wbio
+#define SSL_ST_WBIO 0x20
+
 // ssl_st->s3
 #define SSL_ST_S3 0x30
 
@@ -25,6 +31,9 @@
 // ssl_cipher_st->id
 #define SSL_CIPHER_ST_ID 0x10
 
+// bio_st->num
+#define BIO_ST_NUM 0x18
+
 // bssl::SSL3_STATE->hs
 #define BSSL__SSL3_STATE_HS 0x110
 

kern/openssl.h

@@ -191,21 +191,51 @@ int probe_entry_SSL_write(struct pt_regs* ctx) {
 
     void* ssl = (void*)PT_REGS_PARM1(ctx);
     // https://github.com/openssl/openssl/blob/OpenSSL_1_1_1-stable/crypto/bio/bio_local.h
-    struct ssl_st ssl_info;
-    bpf_probe_read_user(&ssl_info, sizeof(ssl_info), ssl);
 
-    struct BIO bio_w;
-    bpf_probe_read_user(&bio_w, sizeof(bio_w), ssl_info.wbio);
+    u64 *ssl_ver_ptr, *ssl_wbio_ptr, *ssl_wbio_num_ptr;
+    u64 ssl_version, ssl_wbio_addr, ssl_wbio_num_addr;
+    int ret;
+
+    ssl_ver_ptr = (u64 *)(ssl + SSL_ST_VERSION);
+    ret = bpf_probe_read_user(&ssl_version, sizeof(ssl_version),
+                              ssl_ver_ptr);
+    if (ret) {
+        debug_bpf_printk(
+            "(OPENSSL) bpf_probe_read ssl_ver_ptr failed, ret :%d\n",
+            ret);
+        return 0;
+    }
+
+    ssl_wbio_ptr = (u64 *)(ssl + SSL_ST_WBIO);
+    ret = bpf_probe_read_user(&ssl_wbio_addr, sizeof(ssl_wbio_addr),
+                              ssl_wbio_ptr);
+    if (ret) {
+        debug_bpf_printk(
+            "(OPENSSL) bpf_probe_read ssl_wbio_addr failed, ret :%d\n",
+            ret);
+        return 0;
+    }
+
+    // get fd ssl->wbio->num
+    ssl_wbio_num_ptr = (u64 *)(ssl_wbio_ptr + BIO_ST_NUM);
+    ret = bpf_probe_read_user(&ssl_wbio_num_addr, sizeof(ssl_wbio_num_addr),
+                              ssl_wbio_num_ptr);
+    if (ret) {
+        debug_bpf_printk(
+            "(OPENSSL) bpf_probe_read ssl_wbio_num_ptr failed, ret :%d\n",
+            ret);
+        return 0;
+    }
 
     // get fd ssl->wbio->num
-    u32 fd = bio_w.num;
-    debug_bpf_printk("openssl uprobe SSL_write FD:%d\n", fd);
+    u32 fd = (u32)ssl_wbio_num_addr;
+    debug_bpf_printk("openssl uprobe SSL_write FD:%d, version:%d\n", fd, ssl_version);
 
     const char* buf = (const char*)PT_REGS_PARM2(ctx);
     struct active_ssl_buf active_ssl_buf_t;
     __builtin_memset(&active_ssl_buf_t, 0, sizeof(active_ssl_buf_t));
     active_ssl_buf_t.fd = fd;
-    active_ssl_buf_t.version = ssl_info.version;
+    active_ssl_buf_t.version = ssl_version;
     active_ssl_buf_t.buf = buf;
     bpf_map_update_elem(&active_ssl_write_args_map, &current_pid_tgid,
                         &active_ssl_buf_t, BPF_ANY);
@@ -265,21 +295,50 @@ int probe_entry_SSL_read(struct pt_regs* ctx) {
 
     void* ssl = (void*)PT_REGS_PARM1(ctx);
     // https://github.com/openssl/openssl/blob/OpenSSL_1_1_1-stable/crypto/bio/bio_local.h
-    struct ssl_st ssl_info;
-    bpf_probe_read_user(&ssl_info, sizeof(ssl_info), ssl);
+    // Get ssl_rbio pointer
+    u64 *ssl_ver_ptr, *ssl_rbio_ptr, *ssl_rbio_num_ptr;
+    u64 ssl_version, ssl_rbio_addr, ssl_rbio_num_addr;
+    int ret;
+
+    ssl_ver_ptr = (u64 *)(ssl + SSL_ST_VERSION);
+    ret = bpf_probe_read_user(&ssl_version, sizeof(ssl_version),
+                              ssl_ver_ptr);
+    if (ret) {
+        debug_bpf_printk(
+            "(OPENSSL) bpf_probe_read ssl_ver_ptr failed, ret :%d\n",
+            ret);
+        return 0;
+    }
 
-    struct BIO bio_r;
-    bpf_probe_read_user(&bio_r, sizeof(bio_r), ssl_info.rbio);
+    ssl_rbio_ptr = (u64 *)(ssl + SSL_ST_RBIO);
+    ret = bpf_probe_read_user(&ssl_rbio_addr, sizeof(ssl_rbio_addr),
+                              ssl_rbio_ptr);
+    if (ret) {
+        debug_bpf_printk(
+            "(OPENSSL) bpf_probe_read ssl_rbio_ptr failed, ret :%d\n",
+            ret);
+        return 0;
+    }
 
     // get fd ssl->rbio->num
-    u32 fd = bio_r.num;
+    ssl_rbio_num_ptr = (u64 *)(ssl_rbio_addr + BIO_ST_NUM);
+    ret = bpf_probe_read_user(&ssl_rbio_num_addr, sizeof(ssl_rbio_num_addr),
+                              ssl_rbio_num_ptr);
+    if (ret) {
+        debug_bpf_printk(
+            "(OPENSSL) bpf_probe_read ssl_rbio_num_ptr failed, ret :%d\n",
+            ret);
+        return 0;
+    }
+
+    u32 fd = (u32)ssl_rbio_num_addr;
     debug_bpf_printk("openssl uprobe PID:%d, SSL_read FD:%d\n", pid, fd);
 
     const char* buf = (const char*)PT_REGS_PARM2(ctx);
     struct active_ssl_buf active_ssl_buf_t;
     __builtin_memset(&active_ssl_buf_t, 0, sizeof(active_ssl_buf_t));
     active_ssl_buf_t.fd = fd;
-    active_ssl_buf_t.version = ssl_info.version;
+    active_ssl_buf_t.version = ssl_version;
     active_ssl_buf_t.buf = buf;
     bpf_map_update_elem(&active_ssl_read_args_map, &current_pid_tgid,
                         &active_ssl_buf_t, BPF_ANY);

kern/openssl_1_0_2a_kern.c

@@ -13,6 +13,12 @@
 // ssl_st->s3
 #define SSL_ST_S3 0x80
 
+// ssl_st->rbio
+#define SSL_ST_RBIO 0x10
+
+// ssl_st->wbio
+#define SSL_ST_WBIO 0x18
+
 // ssl_session_st->master_key
 #define SSL_SESSION_ST_MASTER_KEY 0x14
 
@@ -28,6 +34,9 @@
 // ssl_cipher_st->id
 #define SSL_CIPHER_ST_ID 0x10
 
+// bio_st->num
+#define BIO_ST_NUM 0x28
+
 // openssl 1.0.2 does not support TLS 1.3, set 0 default
 #define SSL_ST_HANDSHAKE_SECRET 0
 #define SSL_ST_HANDSHAKE_TRAFFIC_HASH 0

kern/openssl_1_1_0a_kern.c

@@ -13,6 +13,12 @@
 // ssl_st->s3
 #define SSL_ST_S3 0x90
 
+// ssl_st->rbio
+#define SSL_ST_RBIO 0x10
+
+// ssl_st->wbio
+#define SSL_ST_WBIO 0x18
+
 // ssl_session_st->master_key
 #define SSL_SESSION_ST_MASTER_KEY 0x8
 
@@ -28,6 +34,9 @@
 // ssl_cipher_st->id
 #define SSL_CIPHER_ST_ID 0x10
 
+// bio_st->num
+#define BIO_ST_NUM 0x28
+
 // openssl 1.1.0 does not support TLS 1.3, set 0 default
 #define SSL_ST_HANDSHAKE_SECRET 0
 #define SSL_ST_HANDSHAKE_TRAFFIC_HASH 0

kern/openssl_1_1_1a_kern.c

@@ -13,6 +13,12 @@
 // ssl_st->s3
 #define SSL_ST_S3 0xa8
 
+// ssl_st->rbio
+#define SSL_ST_RBIO 0x10
+
+// ssl_st->wbio
+#define SSL_ST_WBIO 0x18
+
 // ssl_session_st->master_key
 #define SSL_SESSION_ST_MASTER_KEY 0x50
 
@@ -43,6 +49,9 @@
 // ssl_st->exporter_master_secret
 #define SSL_ST_EXPORTER_MASTER_SECRET 0x3b4
 
+// bio_st->num
+#define BIO_ST_NUM 0x30
+
 #include "openssl.h"
 #include "openssl_masterkey.h"
 

kern/openssl_1_1_1b_kern.c

@@ -13,6 +13,12 @@
 // ssl_st->s3
 #define SSL_ST_S3 0xa8
 
+// ssl_st->rbio
+#define SSL_ST_RBIO 0x10
+
+// ssl_st->wbio
+#define SSL_ST_WBIO 0x18
+
 // ssl_session_st->master_key
 #define SSL_SESSION_ST_MASTER_KEY 0x50
 
@@ -43,6 +49,9 @@
 // ssl_st->exporter_master_secret
 #define SSL_ST_EXPORTER_MASTER_SECRET 0x3b4
 
+// bio_st->num
+#define BIO_ST_NUM 0x30
+
 #include "openssl.h"
 #include "openssl_masterkey.h"
 

kern/openssl_1_1_1d_kern.c

@@ -13,6 +13,12 @@
 // ssl_st->s3
 #define SSL_ST_S3 0xa8
 
+// ssl_st->rbio
+#define SSL_ST_RBIO 0x10
+
+// ssl_st->wbio
+#define SSL_ST_WBIO 0x18
+
 // ssl_session_st->master_key
 #define SSL_SESSION_ST_MASTER_KEY 0x50
 
@@ -43,6 +49,9 @@
 // ssl_st->exporter_master_secret
 #define SSL_ST_EXPORTER_MASTER_SECRET 0x3bc
 
+// bio_st->num
+#define BIO_ST_NUM 0x30
+
 #include "openssl.h"
 #include "openssl_masterkey.h"
 

kern/openssl_1_1_1j_kern.c

@@ -1,8 +1,8 @@
 #ifndef ECAPTURE_OPENSSL_1_1_1_J_KERN_H
 #define ECAPTURE_OPENSSL_1_1_1_J_KERN_H
 
-/* OPENSSL_VERSION_TEXT: OpenSSL 1.1.1s  1 Nov 2022 */
-/* OPENSSL_VERSION_NUMBER: 269488447 */
+/* OPENSSL_VERSION_TEXT: OpenSSL 1.1.1u  30 May 2023 */
+/* OPENSSL_VERSION_NUMBER: 269488479 */
 
 // ssl_st->version
 #define SSL_ST_VERSION 0x0
@@ -13,6 +13,12 @@
 // ssl_st->s3
 #define SSL_ST_S3 0xa8
 
+// ssl_st->rbio
+#define SSL_ST_RBIO 0x10
+
+// ssl_st->wbio
+#define SSL_ST_WBIO 0x18
+
 // ssl_session_st->master_key
 #define SSL_SESSION_ST_MASTER_KEY 0x50
 
@@ -43,6 +49,9 @@
 // ssl_st->exporter_master_secret
 #define SSL_ST_EXPORTER_MASTER_SECRET 0x3bc
 
+// bio_st->num
+#define BIO_ST_NUM 0x30
+
 #include "openssl.h"
 #include "openssl_masterkey.h"
 

kern/openssl_3_0_0_kern.c

@@ -1,8 +1,8 @@
 #ifndef ECAPTURE_OPENSSL_3_0_0_KERN_H
 #define ECAPTURE_OPENSSL_3_0_0_KERN_H
 
-/* OPENSSL_VERSION_TEXT: OpenSSL 3.0.7 1 Nov 2022 */
-/* OPENSSL_VERSION_NUMBER: 805306480 */
+/* OPENSSL_VERSION_TEXT: OpenSSL 3.0.9 30 May 2023 */
+/* OPENSSL_VERSION_NUMBER: 805306512 */
 
 // ssl_st->version
 #define SSL_ST_VERSION 0x0
@@ -13,6 +13,12 @@
 // ssl_st->s3
 #define SSL_ST_S3 0xa8
 
+// ssl_st->rbio
+#define SSL_ST_RBIO 0x10
+
+// ssl_st->wbio
+#define SSL_ST_WBIO 0x18
+
 // ssl_session_st->master_key
 #define SSL_SESSION_ST_MASTER_KEY 0x50
 
@@ -43,6 +49,9 @@
 // ssl_st->exporter_master_secret
 #define SSL_ST_EXPORTER_MASTER_SECRET 0x7c4
 
+// bio_st->num
+#define BIO_ST_NUM 0x38
+
 #include "openssl.h"
 #include "openssl_masterkey_3.0.h"
 

user/module/imodule.go

@@ -302,6 +302,8 @@ func (m *Module) Dispatcher(e event.IEventStruct) {
 	case event.EventTypeModuleData:
 		// Save to cache
 		m.child.Dispatcher(e)
+	default:
+		m.logger.Printf("%s\tunknown event type:%d", m.child.Name(), e.EventType())
 	}
 }
 

user/module/probe_openssl.go

@@ -450,6 +450,10 @@ func (m *MOpenSSLProbe) Events() []*ebpf.Map {
 }
 
 func (m *MOpenSSLProbe) AddConn(pid, fd uint32, addr string) {
+	if fd <= 0 {
+		m.logger.Printf("%s\tAddConn failed. pid:%d, fd:%d, addr:%s\n", m.Name(), pid, fd, addr)
+		return
+	}
 	// save
 	var connMap map[uint32]string
 	var f bool
@@ -459,6 +463,7 @@ func (m *MOpenSSLProbe) AddConn(pid, fd uint32, addr string) {
 	}
 	connMap[fd] = addr
 	m.pidConns[pid] = connMap
+	//m.logger.Printf("%s\tAddConn pid:%d, fd:%d, addr:%s, mapinfo:%v\n", m.Name(), pid, fd, addr, m.pidConns)
 	return
 }
 
@@ -484,9 +489,13 @@ func (m *MOpenSSLProbe) DelConn(pid, fd uint32) {
 	return
 }
 func (m *MOpenSSLProbe) GetConn(pid, fd uint32) string {
+	if fd <= 0 {
+		return ConnNotFound
+	}
 	addr := ""
 	var connMap map[uint32]string
 	var f bool
+	//m.logger.Printf("%s\tGetConn pid:%d, fd:%d, mapinfo:%v\n", m.Name(), pid, fd, m.pidConns)
 	connMap, f = m.pidConns[pid]
 	if !f {
 		return ConnNotFound
@@ -701,7 +710,11 @@ func (m *MOpenSSLProbe) Dispatcher(eventStruct event.IEventStruct) {
 }
 
 func (m *MOpenSSLProbe) dumpSslData(eventStruct *event.SSLDataEvent) {
+	if eventStruct.Fd <= 0 {
+		m.logger.Printf("\tnotice: SSLDataEvent's fd is 0.  pid:%d, fd:%d, addr:%s\n", eventStruct.Pid, eventStruct.Fd, eventStruct.Addr)
+	}
 	var addr = m.GetConn(eventStruct.Pid, eventStruct.Fd)
+	//m.logger.Printf("\tSSLDataEvent pid:%d, fd:%d, addr:%s\n", eventStruct.Pid, eventStruct.Fd, addr)
 	if addr == ConnNotFound {
 		eventStruct.Addr = DefaultAddr
 	} else {

utils/boringssl-offset.c

@@ -13,6 +13,7 @@
 // limitations under the License.
 
 //  g++ -I include/ -I src/ ./src/offset.c -o off
+#include <ctype.h>
 #include <openssl/base.h>
 #include <openssl/crypto.h>
 #include <ssl/internal.h>
@@ -22,10 +23,13 @@
 #define SSL_STRUCT_OFFSETS                   \
     X(ssl_st, version)                       \
     X(ssl_st, session)                       \
+    X(ssl_st, rbio)                          \
+    X(ssl_st, wbio)                          \
     X(ssl_st, s3)                            \
     X(ssl_session_st, secret_length)         \
     X(ssl_session_st, secret)                \
     X(ssl_session_st, cipher)                \
+    X(bio_st, num)                           \
     X(ssl_cipher_st, id)                     \
     X(bssl::SSL3_STATE, hs)                  \
     X(bssl::SSL3_STATE, client_random)       \