From dd0c48acadfe635a01d03d797c57927cd760b323 Mon Sep 17 00:00:00 2001 From: CFC4N Date: Sun, 24 Mar 2024 19:31:14 +0800 Subject: [PATCH 1/4] Fix cant found RET offset in gotls mode. fix #502. Fix the issue of not being able to fetch the function RET offset in the gotls model when building a Golang binary with pie mode. Signed-off-by: CFC4N --- builder/Makefile.release | 28 ++-- user/config/config_gotls.go | 229 ++++++++++++++++++++++----- user/config/go_instructions_amd64.go | 21 +++ user/config/go_instructions_arm64.go | 23 +++ user/module/probe_gotls.go | 7 +- user/module/probe_gotls_keylog.go | 6 +- user/module/probe_gotls_pcap.go | 5 +- user/module/probe_gotls_text.go | 8 +- 8 files changed, 259 insertions(+), 68 deletions(-) create mode 100644 user/config/go_instructions_amd64.go create mode 100644 user/config/go_instructions_arm64.go diff --git a/builder/Makefile.release b/builder/Makefile.release index 94e784889..d45a4e73d 100644 --- a/builder/Makefile.release +++ b/builder/Makefile.release @@ -50,7 +50,9 @@ endif OUTPUT_DIR = ./bin TAR_DIR = ecapture-$(SNAPSHOT_VERSION)-linux-$(UNAME_M) +TAR_DIR_NOCORE = ecapture-$(SNAPSHOT_VERSION)-linux-nocore TAR_DIR_ANDROID = ecapture-$(SNAPSHOT_VERSION)-android-$(UNAME_M) +TAR_DIR_ANDROID_NOCORE = ecapture-$(SNAPSHOT_VERSION)-android-$(UNAME_M)-nocore # from CLI args. RELEASE_NOTES ?= $(OUTPUT_DIR)/release_notes.txt @@ -71,7 +73,9 @@ BUILD_DIR = build # OUT_ARCHIVE := $(OUTPUT_DIR)/$(TAR_DIR).tar.gz -OUT_ARCHIVE_ANDROID := $(OUTPUT_DIR)/$(TAR_DIR_ANDROID)-nocore.tar.gz +OUT_ARCHIVE_NOCORE := $(OUTPUT_DIR)/$(TAR_DIR_NOCORE).tar.gz +OUT_ARCHIVE_ANDROID := $(OUTPUT_DIR)/$(TAR_DIR_ANDROID).tar.gz +OUT_ARCHIVE_ANDROID_NOCORE := $(OUTPUT_DIR)/$(TAR_DIR_ANDROID_NOCORE).tar.gz OUT_CHECKSUMS := $(OUTPUT_DIR)/checksum-$(SNAPSHOT_VERSION).txt @@ -183,7 +187,7 @@ snapshot: \ $(CMD_CP) $(OUTPUT_DIR)/ecapture $(TAR_DIR)/ecapture $(CMD_TAR) -czf $(OUT_ARCHIVE) $(TAR_DIR) cd $(OUTPUT_DIR) - $(CMD_CHECKSUM) $(TAR_DIR).tar.gz > ./../$(OUT_CHECKSUMS) + $(CMD_CHECKSUM) $(OUT_ARCHIVE_ANDROID) > ./../$(OUT_CHECKSUMS) cd ../ .PHONY: snapshot_android @@ -198,29 +202,29 @@ snapshot_android: \ $(MAKE) clean ANDROID=1 $(MAKE) nocore RELEASE_TAG=$(SNAPSHOT_VERSION) # create the tar ball and checksum files - $(CMD_MKDIR) -p $(TAR_DIR_ANDROID) - $(CMD_CP) LICENSE $(TAR_DIR_ANDROID)/LICENSE - $(CMD_CP) CHANGELOG.md $(TAR_DIR_ANDROID)/CHANGELOG.md - $(CMD_CP) README.md $(TAR_DIR_ANDROID)/README.md - $(CMD_CP) README_CN.md $(TAR_DIR_ANDROID)/README_CN.md - $(CMD_CP) $(OUTPUT_DIR)/ecapture $(TAR_DIR_ANDROID)/ecapture - $(CMD_TAR) -czf $(OUT_ARCHIVE_ANDROID) $(TAR_DIR_ANDROID) + $(CMD_MKDIR) -p $(TAR_DIR_ANDROID_NOCORE) + $(CMD_CP) LICENSE $(TAR_DIR_ANDROID_NOCORE)/LICENSE + $(CMD_CP) CHANGELOG.md $(TAR_DIR_ANDROID_NOCORE)/CHANGELOG.md + $(CMD_CP) README.md $(TAR_DIR_ANDROID_NOCORE)/README.md + $(CMD_CP) README_CN.md $(TAR_DIR_ANDROID_NOCORE)/README_CN.md + $(CMD_CP) $(OUTPUT_DIR)/ecapture $(TAR_DIR_ANDROID_NOCORE)/ecapture + $(CMD_TAR) -czf $(OUT_ARCHIVE_ANDROID_NOCORE) $(TAR_DIR_ANDROID_NOCORE) cd $(OUTPUT_DIR) - $(CMD_CHECKSUM) $(TAR_DIR_ANDROID).tar.gz >> ./../$(OUT_CHECKSUMS) + $(CMD_CHECKSUM) $(OUT_ARCHIVE_ANDROID_NOCORE) >> ./../$(OUT_CHECKSUMS) cd ../ .PHONY: publish publish: \ $(OUTPUT_DIR) \ $(OUT_ARCHIVE) \ - $(OUT_ARCHIVE_ANDROID) \ + $(OUT_ARCHIVE_ANDROID_NOCORE) \ $(OUT_CHECKSUMS) \ $(OUT_DEB_FILE) \ | .check_tree \ .check_$(CMD_GITHUB) # # release it! - $(CMD_GITHUB) release create $(SNAPSHOT_VERSION) $(OUT_ARCHIVE) $(OUT_ARCHIVE_ANDROID) $(OUT_DEB_FILE) $(OUT_CHECKSUMS) --title "eCapture $(SNAPSHOT_VERSION)" --notes-file $(RELEASE_NOTES) + $(CMD_GITHUB) release create $(SNAPSHOT_VERSION) $(OUT_ARCHIVE) $(OUT_ARCHIVE_ANDROID_NOCORE) $(OUT_DEB_FILE) $(OUT_CHECKSUMS) --title "eCapture $(SNAPSHOT_VERSION)" --notes-file $(RELEASE_NOTES) .PHONY: clean clean: diff --git a/user/config/config_gotls.go b/user/config/config_gotls.go index 3f44c3131..c35a78634 100644 --- a/user/config/config_gotls.go +++ b/user/config/config_gotls.go @@ -15,41 +15,85 @@ package config import ( + "bytes" + "debug/buildinfo" "debug/elf" + "debug/gosym" + "encoding/binary" "errors" "fmt" "os" "runtime" - - "golang.org/x/arch/arm64/arm64asm" - "golang.org/x/arch/x86/x86asm" + "strings" ) const ( - // Arm64armInstSize via : arm64/arm64asm/decode.go:Decode() size = 4 - Arm64armInstSize = 4 - + // 46EE50 - 46EDB0 + // GoTlsReadFunc = "crypto/tls.(*Conn).Read" + + GoTlsWriteFunc = "crypto/tls.(*Conn).writeRecordLocked" + GoTlsMasterSecretFunc = "crypto/tls.(*Config).writeKeyLog" + // crypto_tls._ptr_Conn.Read + GoTlsReadFuncArm64 = "crypto_tls._ptr_Conn.Read" ) var ( - ErrorGoBINNotFound = errors.New("The executable program (compiled by Golang) was not found") - ErrorSymbolNotFound = errors.New("symbol not found") - ErrorNoRetFound = errors.New("no RET instructions found") + ErrorGoBINNotFound = errors.New("The executable program (compiled by Golang) was not found") + ErrorSymbolEmpty = errors.New("symbol is empty") + ErrorSymbolNotFound = errors.New("symbol not found") + ErrorSymbolNotFoundFromTable = errors.New("symbol not found from table") + ErrorNoRetFound = errors.New("no RET instructions found") + ErrorNoRetFoundFromSymTabFun = errors.New("no RET instructions found from golang symbol table with Fun") ) +// From go/src/debug/gosym/pclntab.go +const ( + go12magic = 0xfffffffb + go116magic = 0xfffffffa + go118magic = 0xfffffff0 + go120magic = 0xfffffff1 +) + +// Select the magic number based on the Go version +func magicNumber(goVersion string) []byte { + bs := make([]byte, 4) + var magic uint32 + if strings.Compare(goVersion, "go1.20") >= 0 { + magic = go120magic + } else if strings.Compare(goVersion, "go1.18") >= 0 { + magic = go118magic + } else if strings.Compare(goVersion, "go1.16") >= 0 { + magic = go116magic + } else { + magic = go12magic + } + binary.LittleEndian.PutUint32(bs, magic) + return bs +} + +type FuncOffsets struct { + Start uint64 + Returns []uint64 +} + // GoTLSConfig represents configuration for Go SSL probe type GoTLSConfig struct { eConfig - Path string `json:"path"` // golang application path to binary built with Go toolchain. - PcapFile string `json:"pcapFile"` // pcapFile the raw packets to file rather than parsing and printing them out. - KeylogFile string `json:"keylogFile"` // keylogFile The file stores SSL/TLS keys, and eCapture captures these keys during encrypted traffic communication and saves them to the file. - Model string `json:"model"` // model such as : text, pcapng/pcap, key/keylog. - Ifname string `json:"ifName"` // (TC Classifier) Interface name on which the probe will be attached. - PcapFilter string `json:"pcapFilter"` // pcap filter - goElfArch string // - goElf *elf.File // - ReadTlsAddrs []int + Path string `json:"path"` // golang application path to binary built with Go toolchain. + PcapFile string `json:"pcapFile"` // pcapFile the raw packets to file rather than parsing and printing them out. + KeylogFile string `json:"keylogFile"` // keylogFile The file stores SSL/TLS keys, and eCapture captures these keys during encrypted traffic communication and saves them to the file. + Model string `json:"model"` // model such as : text, pcapng/pcap, key/keylog. + Ifname string `json:"ifName"` // (TC Classifier) Interface name on which the probe will be attached. + PcapFilter string `json:"pcapFilter"` // pcap filter + goElfArch string // + goElf *elf.File // + buildinfo *buildinfo.BuildInfo + ReadTlsAddrs []int + GoTlsWriteAddr uint64 + GoTlsMasterSecretAddr uint64 + IsPieBuildMode bool + goSymTab *gosym.Table } // NewGoTLSConfig creates a new config for Go SSL @@ -74,6 +118,12 @@ func (gc *GoTLSConfig) Check() error { return err } + // Read the build information of the Go application + gc.buildinfo, err = buildinfo.ReadFile(gc.Path) + if err != nil { + return err + } + var goElf *elf.File goElf, err = elf.Open(gc.Path) if err != nil { @@ -102,10 +152,56 @@ func (gc *GoTLSConfig) Check() error { } gc.goElfArch = goElfArch gc.goElf = goElf - gc.ReadTlsAddrs, err = gc.findRetOffsets(GoTlsReadFunc) + // If built with PIE and stripped, gopclntab is + // unlabeled and nested under .data.rel.ro. + for _, bs := range gc.buildinfo.Settings { + if bs.Key == "-buildmode" { + if bs.Value == "pie" { + gc.IsPieBuildMode = true + } + break + } + } + if gc.IsPieBuildMode { + gc.goSymTab, err = gc.ReadTable() + if err != nil { + return err + } + fun := gc.goSymTab.LookupFunc(GoTlsWriteFunc) + if fun != nil { + gc.GoTlsWriteAddr = fun.Entry + } + fun = gc.goSymTab.LookupFunc(GoTlsMasterSecretFunc) + if fun != nil { + gc.GoTlsMasterSecretAddr = fun.Entry + } + gc.ReadTlsAddrs, err = gc.findRetOffsetsPie(GoTlsReadFunc) + if err != nil { + return err + } + } else { + gc.ReadTlsAddrs, err = gc.findRetOffsets(GoTlsReadFunc) + if err != nil { + return err + } + } return err } +func (gc *GoTLSConfig) findRetOffsetsPie(symbolName string) ([]int, error) { + var offsets []int + var err error + + fun := gc.goSymTab.LookupFunc(symbolName) + if fun == nil { + return nil, ErrorSymbolNotFoundFromTable + } + + //fmt.Printf("found in %s, entry:%x, end:%x , end-entry:%x\n", fun.Name, fun.Entry, fun.End, fun.End-fun.Entry) + offsets, err = gc.findFuncOffsetBySymfun(fun, gc.goElf) + return offsets, err +} + // FindRetOffsets searches for the addresses of all RET instructions within // the instruction set associated with the specified symbol in an ELF program. // It is used for mounting uretprobe programs for Golang programs, @@ -125,7 +221,7 @@ func (gc *GoTLSConfig) findRetOffsets(symbolName string) ([]int, error) { } if len(allSymbs) == 0 { - return nil, fmt.Errorf("no symbols found") + return nil, ErrorSymbolEmpty } var found bool @@ -163,30 +259,6 @@ func (gc *GoTLSConfig) findRetOffsets(symbolName string) ([]int, error) { return offsets, nil } -// decodeInstruction Decode into assembly instructions and identify the RET instruction to return the offset. -func (gc *GoTLSConfig) decodeInstruction(instHex []byte) ([]int, error) { - var offsets []int - for i := 0; i < len(instHex); { - if gc.goElfArch == "amd64" { - inst, err := x86asm.Decode(instHex[i:], 64) - if err != nil { - return nil, err - } - if inst.Op == x86asm.RET { - offsets = append(offsets, i) - } - i += inst.Len - } else { - inst, _ := arm64asm.Decode(instHex[i:]) // Why ignore error: https://github.com/gojue/ecapture/pull/506 - if inst.Op == arm64asm.RET { - offsets = append(offsets, i) - } - i += Arm64armInstSize - } - } - return offsets, nil -} - func (gc *GoTLSConfig) checkModel() (string, error) { var m string var e error @@ -204,3 +276,72 @@ func (gc *GoTLSConfig) checkModel() (string, error) { } return m, e } + +func (gc *GoTLSConfig) ReadTable() (*gosym.Table, error) { + sectionLabel := ".data.rel.ro" + section := gc.goElf.Section(sectionLabel) + if section == nil { + // binary may be built with -pie + sectionLabel = ".gopclntab" + section = gc.goElf.Section(sectionLabel) + if section == nil { + return nil, fmt.Errorf("could not read section %s from %s ", sectionLabel, gc.Path) + } + } + tableData, err := section.Data() + if err != nil { + return nil, fmt.Errorf("found section but could not read %s from %s ", sectionLabel, gc.Path) + } + + // Find .gopclntab by magic number even if there is no section label + magic := magicNumber(gc.buildinfo.GoVersion) + pclntabIndex := bytes.Index(tableData, magic) + //fmt.Printf("buildinfo :%v, magic:%x, pclntabIndex:%d offset:%x , section:%v \n", gc.buildinfo, magic, pclntabIndex, section.Offset, section) + if pclntabIndex < 0 { + return nil, fmt.Errorf("could not find magic number in %s ", gc.Path) + } + tableData = tableData[pclntabIndex:] + addr := gc.goElf.Section(".text").Addr + lineTable := gosym.NewLineTable(tableData, addr) + symTable, err := gosym.NewTable([]byte{}, lineTable) + if err != nil { + return nil, ErrorSymbolNotFoundFromTable + } + return symTable, nil +} + +func (gc *GoTLSConfig) findFuncOffsetBySymfun(f *gosym.Func, elfF *elf.File) ([]int, error) { + var offsets []int + var err error + + for _, prog := range elfF.Progs { + if prog.Type != elf.PT_LOAD || (prog.Flags&elf.PF_X) == 0 { + continue + } + // For more info on this calculation: stackoverflow.com/a/40249502 + /* + ida : start : 46EE50, end : 46F258 , len :0x408 + elf info: start : 46ed30, end : , len: 0x3B0, 0x58 + + */ + if prog.Vaddr <= f.Value && f.Value < (prog.Vaddr+prog.Memsz) { + funcLen := f.End - f.Entry + fmt.Printf("name :%s, f.Value:%x, f.Entry:%x, f.End:%x, funcLen:%X \n ", f.Name, f.Value, f.Entry, f.End, funcLen) + data := make([]byte, funcLen) + _, err = prog.ReadAt(data, int64(f.Value-prog.Vaddr)) + if err != nil { + return offsets, fmt.Errorf("finding function return: %w", err) + } + + offsets, err = gc.decodeInstruction(data) + if err != nil { + return offsets, fmt.Errorf("finding function return: %w", err) + } + for i, offset := range offsets { + offsets[i] = int(f.Entry) + offset + } + return offsets, nil + } + } + return offsets, ErrorNoRetFoundFromSymTabFun +} diff --git a/user/config/go_instructions_amd64.go b/user/config/go_instructions_amd64.go new file mode 100644 index 000000000..f2d9c7c8d --- /dev/null +++ b/user/config/go_instructions_amd64.go @@ -0,0 +1,21 @@ +package config + +import ( + "golang.org/x/arch/x86/x86asm" +) + +// decodeInstruction Decode into assembly instructions and identify the RET instruction to return the offset. +func (gc *GoTLSConfig) decodeInstruction(instHex []byte) ([]int, error) { + var offsets []int + for i := 0; i < len(instHex); { + inst, err := x86asm.Decode(instHex[i:], 64) + if err != nil { + return nil, err + } + if inst.Op == x86asm.RET { + offsets = append(offsets, i) + } + i += inst.Len + } + return offsets, nil +} diff --git a/user/config/go_instructions_arm64.go b/user/config/go_instructions_arm64.go new file mode 100644 index 000000000..ac4c20a6b --- /dev/null +++ b/user/config/go_instructions_arm64.go @@ -0,0 +1,23 @@ +package config + +import ( + "golang.org/x/arch/arm64/arm64asm" +) + +const ( + // Arm64armInstSize via : arm64/arm64asm/decode.go:Decode() size = 4 + Arm64armInstSize = 4 +) + +// decodeInstruction Decode into assembly instructions and identify the RET instruction to return the offset. +func (gc *GoTLSConfig) decodeInstruction(instHex []byte) ([]int, error) { + var offsets []int + for i := 0; i < len(instHex); { + inst, _ := arm64asm.Decode(instHex[i:]) // Why ignore error: https://github.com/gojue/ecapture/pull/506 + if inst.Op == arm64asm.RET { + offsets = append(offsets, i) + } + i += Arm64armInstSize + } + return offsets, nil +} diff --git a/user/module/probe_gotls.go b/user/module/probe_gotls.go index c11479316..d7ad4b65e 100644 --- a/user/module/probe_gotls.go +++ b/user/module/probe_gotls.go @@ -29,11 +29,6 @@ func init() { Register(mod) } -const ( - goTlsWriteFunc = "crypto/tls.(*Conn).writeRecordLocked" - goTlsMasterSecretFunc = "crypto/tls.(*Config).writeKeyLog" -) - var NotGoCompiledBin = errors.New("It is not a program compiled in the Go language.") // GoTLSProbe represents a probe for Go SSL @@ -261,6 +256,8 @@ func (g *GoTLSProbe) saveMasterSecret(secretEvent *event.MasterSecretGotlsEvent) g.logger.Fatalf("%s: save masterSecrets to pcapng error:%s", secretEvent.String(), e.Error()) return } + default: + g.logger.Fatalf("unhandled default case with eBPF Program type:%d", g.eBPFProgramType) } } diff --git a/user/module/probe_gotls_keylog.go b/user/module/probe_gotls_keylog.go index 244e5fd73..3d55a5794 100644 --- a/user/module/probe_gotls_keylog.go +++ b/user/module/probe_gotls_keylog.go @@ -15,6 +15,7 @@ package module import ( + "ecapture/user/config" "ecapture/user/event" "errors" "github.com/cilium/ebpf" @@ -26,7 +27,7 @@ import ( func (g *GoTLSProbe) setupManagersKeylog() error { g.logger.Printf("%s\tHOOK type:golang elf, binrayPath:%s\n", g.Name(), g.path) - g.logger.Printf("%s\tHook masterKey function:%s\n", g.Name(), goTlsMasterSecretFunc) + g.logger.Printf("%s\tHook masterKey function:%s, Address:%x \n", g.Name(), config.GoTlsMasterSecretFunc, g.conf.(*config.GoTLSConfig).GoTlsMasterSecretAddr) var ( sec string @@ -47,9 +48,10 @@ func (g *GoTLSProbe) setupManagersKeylog() error { { Section: sec, EbpfFuncName: fn, - AttachToFuncName: goTlsMasterSecretFunc, + AttachToFuncName: config.GoTlsMasterSecretFunc, BinaryPath: g.path, UID: "uprobe_gotls_master_secret", + UAddress: g.conf.(*config.GoTLSConfig).GoTlsMasterSecretAddr, }, }, diff --git a/user/module/probe_gotls_pcap.go b/user/module/probe_gotls_pcap.go index 2f80813c9..3d65f382f 100644 --- a/user/module/probe_gotls_pcap.go +++ b/user/module/probe_gotls_pcap.go @@ -55,7 +55,7 @@ func (g *GoTLSProbe) setupManagersPcap() error { g.logger.Printf("%s\tHOOK type:golang elf, binrayPath:%s\n", g.Name(), g.path) g.logger.Printf("%s\tPcapFilter: %s\n", g.Name(), pcapFilter) g.logger.Printf("%s\tIfname: %s, Ifindex: %d\n", g.Name(), g.ifName, g.ifIdex) - g.logger.Printf("%s\tHook masterKey function: %s\n", g.Name(), goTlsMasterSecretFunc) + g.logger.Printf("%s\tHook masterKey function: %s, Address:%x \n", g.Name(), config.GoTlsMasterSecretFunc, g.conf.(*config.GoTLSConfig).GoTlsMasterSecretAddr) // create pcapng writer netIfs, err := net.Interfaces() @@ -106,9 +106,10 @@ func (g *GoTLSProbe) setupManagersPcap() error { { Section: sec, EbpfFuncName: fn, - AttachToFuncName: goTlsMasterSecretFunc, + AttachToFuncName: config.GoTlsMasterSecretFunc, BinaryPath: g.path, UID: "uprobe_gotls_master_secret", + UAddress: g.conf.(*config.GoTLSConfig).GoTlsMasterSecretAddr, }, }, diff --git a/user/module/probe_gotls_text.go b/user/module/probe_gotls_text.go index 2fe0112fd..0a262293a 100644 --- a/user/module/probe_gotls_text.go +++ b/user/module/probe_gotls_text.go @@ -48,8 +48,9 @@ func (g *GoTLSProbe) setupManagersText() error { { Section: sec, EbpfFuncName: fn, - AttachToFuncName: goTlsWriteFunc, + AttachToFuncName: config.GoTlsWriteFunc, BinaryPath: g.path, + UAddress: g.conf.(*config.GoTLSConfig).GoTlsWriteAddr, }, }, Maps: []*manager.Map{ @@ -69,8 +70,9 @@ func (g *GoTLSProbe) setupManagersText() error { EbpfFuncName: readFn, AttachToFuncName: config.GoTlsReadFunc, BinaryPath: g.path, - UprobeOffset: uint64(v), - UID: uid, + //UprobeOffset: uint64(v), + UAddress: uint64(v), + UID: uid, }) } g.bpfManagerOptions = manager.Options{ From d53387f8dd313d27f7433d57242eee6a73c9453d Mon Sep 17 00:00:00 2001 From: CFC4N Date: Sat, 30 Mar 2024 18:35:48 +0800 Subject: [PATCH 2/4] support symbols in sections `.data.rel.ro.gopclntab` and `.data.rel.ro`. see PR #512 for more info . Signed-off-by: CFC4N --- kern/gotls_kern.c | 2 +- user/config/config_gotls.go | 146 ++++++++++++++++++------------ user/event/event_gotls.go | 2 + user/module/probe_gotls_keylog.go | 22 ++++- user/module/probe_gotls_text.go | 4 + 5 files changed, 116 insertions(+), 60 deletions(-) diff --git a/kern/gotls_kern.c b/kern/gotls_kern.c index f109d8fc2..0e7a3006c 100644 --- a/kern/gotls_kern.c +++ b/kern/gotls_kern.c @@ -251,7 +251,7 @@ static __always_inline int gotls_mastersecret(struct pt_regs *ctx, return 0; } - debug_bpf_printk("gotls_mastersecret read mastersecret label%s\n", + debug_bpf_printk("gotls_mastersecret read mastersecret label:%s\n", mastersecret_gotls.label); ret = bpf_probe_read_user_str(&mastersecret_gotls.client_random, sizeof(mastersecret_gotls.client_random), diff --git a/user/config/config_gotls.go b/user/config/config_gotls.go index c35a78634..7e9608074 100644 --- a/user/config/config_gotls.go +++ b/user/config/config_gotls.go @@ -28,14 +28,15 @@ import ( ) const ( - // 46EE50 - 46EDB0 - // - GoTlsReadFunc = "crypto/tls.(*Conn).Read" - + GoTlsReadFunc = "crypto/tls.(*Conn).Read" GoTlsWriteFunc = "crypto/tls.(*Conn).writeRecordLocked" GoTlsMasterSecretFunc = "crypto/tls.(*Config).writeKeyLog" - // crypto_tls._ptr_Conn.Read - GoTlsReadFuncArm64 = "crypto_tls._ptr_Conn.Read" + + /* + 我是通过IDA静态分析符号发现`crypto/tls.(*Conn).Read`的地址是`46EE50`。用程序计算出来的总是比这个数字少了`0x120` ，通过分析其他多个编译的程序，发现差值总是`0x120`。 + 所以，我定义了一个常量，增加到程序计算的地址上。但是我不知道原因，如果你知道，请告诉我。更多信息见：https://github.com/gojue/ecapture/pull/512 + */ + IdaProOffset = 0x120 ) var ( @@ -88,7 +89,7 @@ type GoTLSConfig struct { PcapFilter string `json:"pcapFilter"` // pcap filter goElfArch string // goElf *elf.File // - buildinfo *buildinfo.BuildInfo + Buildinfo *buildinfo.BuildInfo ReadTlsAddrs []int GoTlsWriteAddr uint64 GoTlsMasterSecretAddr uint64 @@ -119,7 +120,7 @@ func (gc *GoTLSConfig) Check() error { } // Read the build information of the Go application - gc.buildinfo, err = buildinfo.ReadFile(gc.Path) + gc.Buildinfo, err = buildinfo.ReadFile(gc.Path) if err != nil { return err } @@ -154,7 +155,7 @@ func (gc *GoTLSConfig) Check() error { gc.goElf = goElf // If built with PIE and stripped, gopclntab is // unlabeled and nested under .data.rel.ro. - for _, bs := range gc.buildinfo.Settings { + for _, bs := range gc.Buildinfo.Settings { if bs.Key == "-buildmode" { if bs.Value == "pie" { gc.IsPieBuildMode = true @@ -167,16 +168,21 @@ func (gc *GoTLSConfig) Check() error { if err != nil { return err } - fun := gc.goSymTab.LookupFunc(GoTlsWriteFunc) - if fun != nil { - gc.GoTlsWriteAddr = fun.Entry + var addr uint64 + addr, err = gc.findPieSymbolAddr(GoTlsWriteFunc) + if err != nil { + return fmt.Errorf("%s symbol address error:%s", GoTlsWriteFunc, err.Error()) } - fun = gc.goSymTab.LookupFunc(GoTlsMasterSecretFunc) - if fun != nil { - gc.GoTlsMasterSecretAddr = fun.Entry + gc.GoTlsWriteAddr = addr + addr, err = gc.findPieSymbolAddr(GoTlsMasterSecretFunc) + if err != nil { + return fmt.Errorf("%s symbol address error:%s", GoTlsMasterSecretFunc, err.Error()) } + gc.GoTlsMasterSecretAddr = addr + gc.ReadTlsAddrs, err = gc.findRetOffsetsPie(GoTlsReadFunc) if err != nil { + panic(err) return err } } else { @@ -188,20 +194,6 @@ func (gc *GoTLSConfig) Check() error { return err } -func (gc *GoTLSConfig) findRetOffsetsPie(symbolName string) ([]int, error) { - var offsets []int - var err error - - fun := gc.goSymTab.LookupFunc(symbolName) - if fun == nil { - return nil, ErrorSymbolNotFoundFromTable - } - - //fmt.Printf("found in %s, entry:%x, end:%x , end-entry:%x\n", fun.Name, fun.Entry, fun.End, fun.End-fun.Entry) - offsets, err = gc.findFuncOffsetBySymfun(fun, gc.goElf) - return offsets, err -} - // FindRetOffsets searches for the addresses of all RET instructions within // the instruction set associated with the specified symbol in an ELF program. // It is used for mounting uretprobe programs for Golang programs, @@ -214,7 +206,6 @@ func (gc *GoTLSConfig) findRetOffsets(symbolName string) ([]int, error) { if len(goSymbs) > 0 { allSymbs = append(allSymbs, goSymbs...) } - goDynamicSymbs, _ := gc.goElf.DynamicSymbols() if len(goDynamicSymbs) > 0 { allSymbs = append(allSymbs, goDynamicSymbs...) @@ -256,6 +247,23 @@ func (gc *GoTLSConfig) findRetOffsets(symbolName string) ([]int, error) { if len(offsets) == 0 { return offsets, ErrorNoRetFound } + + address := symbol.Value + for _, prog := range gc.goElf.Progs { + // Skip uninteresting segments. + if prog.Type != elf.PT_LOAD || (prog.Flags&elf.PF_X) == 0 { + continue + } + + if prog.Vaddr <= symbol.Value && symbol.Value < (prog.Vaddr+prog.Memsz) { + // stackoverflow.com/a/40249502 + address = symbol.Value - prog.Vaddr + prog.Off + break + } + } + for i, offset := range offsets { + offsets[i] = int(address) + offset + } return offsets, nil } @@ -278,29 +286,33 @@ func (gc *GoTLSConfig) checkModel() (string, error) { } func (gc *GoTLSConfig) ReadTable() (*gosym.Table, error) { - sectionLabel := ".data.rel.ro" + sectionLabel := ".gopclntab" section := gc.goElf.Section(sectionLabel) if section == nil { // binary may be built with -pie - sectionLabel = ".gopclntab" + sectionLabel = ".data.rel.ro.gopclntab" section = gc.goElf.Section(sectionLabel) if section == nil { - return nil, fmt.Errorf("could not read section %s from %s ", sectionLabel, gc.Path) + sectionLabel = ".data.rel.ro" + section = gc.goElf.Section(sectionLabel) + if section == nil { + return nil, fmt.Errorf("could not read section %s from %s ", sectionLabel, gc.Path) + } } } tableData, err := section.Data() if err != nil { return nil, fmt.Errorf("found section but could not read %s from %s ", sectionLabel, gc.Path) } - // Find .gopclntab by magic number even if there is no section label - magic := magicNumber(gc.buildinfo.GoVersion) + magic := magicNumber(gc.Buildinfo.GoVersion) pclntabIndex := bytes.Index(tableData, magic) - //fmt.Printf("buildinfo :%v, magic:%x, pclntabIndex:%d offset:%x , section:%v \n", gc.buildinfo, magic, pclntabIndex, section.Offset, section) + //fmt.Printf("Buildinfo :%v, magic:%x, pclntabIndex:%d offset:%x , section:%v \n", gc.Buildinfo, magic, pclntabIndex, section.Offset, section) if pclntabIndex < 0 { return nil, fmt.Errorf("could not find magic number in %s ", gc.Path) } tableData = tableData[pclntabIndex:] + addr := gc.goElf.Section(".text").Addr lineTable := gosym.NewLineTable(tableData, addr) symTable, err := gosym.NewTable([]byte{}, lineTable) @@ -310,38 +322,60 @@ func (gc *GoTLSConfig) ReadTable() (*gosym.Table, error) { return symTable, nil } -func (gc *GoTLSConfig) findFuncOffsetBySymfun(f *gosym.Func, elfF *elf.File) ([]int, error) { +func (gc *GoTLSConfig) findRetOffsetsPie(lfunc string) ([]int, error) { var offsets []int + var address uint64 var err error + address, err = gc.findPieSymbolAddr(lfunc) + if err != nil { + return offsets, err + } + f := gc.goSymTab.LookupFunc(lfunc) + funcLen := f.End - f.Entry + for _, prog := range gc.goElf.Progs { + if prog.Type != elf.PT_LOAD || (prog.Flags&elf.PF_X) == 0 { + continue + } + data := make([]byte, funcLen) + _, err = prog.ReadAt(data, int64(address)) + if err != nil { + return offsets, fmt.Errorf("finding function return: %w", err) + } + offsets, err = gc.decodeInstruction(data) + if err != nil { + return offsets, fmt.Errorf("finding function return: %w", err) + } + for i, offset := range offsets { + offsets[i] = int(address) + offset + } + return offsets, nil + } + return offsets, errors.New("cant found gotls symbol offsets.") +} - for _, prog := range elfF.Progs { +func (gc *GoTLSConfig) findPieSymbolAddr(lfunc string) (uint64, error) { + f := gc.goSymTab.LookupFunc(lfunc) + if f == nil { + return 0, errors.New("Cant found symbol address on pie model.") + } + var err error + var address uint64 + for _, prog := range gc.goElf.Progs { if prog.Type != elf.PT_LOAD || (prog.Flags&elf.PF_X) == 0 { continue } // For more info on this calculation: stackoverflow.com/a/40249502 - /* - ida : start : 46EE50, end : 46F258 , len :0x408 - elf info: start : 46ed30, end : , len: 0x3B0, 0x58 - - */ + address = f.Value if prog.Vaddr <= f.Value && f.Value < (prog.Vaddr+prog.Memsz) { funcLen := f.End - f.Entry - fmt.Printf("name :%s, f.Value:%x, f.Entry:%x, f.End:%x, funcLen:%X \n ", f.Name, f.Value, f.Entry, f.End, funcLen) data := make([]byte, funcLen) - _, err = prog.ReadAt(data, int64(f.Value-prog.Vaddr)) + address = f.Value - prog.Vaddr + prog.Off + IdaProOffset + _, err = prog.ReadAt(data, int64(address)) if err != nil { - return offsets, fmt.Errorf("finding function return: %w", err) - } - - offsets, err = gc.decodeInstruction(data) - if err != nil { - return offsets, fmt.Errorf("finding function return: %w", err) - } - for i, offset := range offsets { - offsets[i] = int(f.Entry) + offset + return 0, fmt.Errorf("search function return: %w", err) } - return offsets, nil + return address, nil } } - return offsets, ErrorNoRetFoundFromSymTabFun + return 0, ErrorNoRetFoundFromSymTabFun } diff --git a/user/event/event_gotls.go b/user/event/event_gotls.go index ae1392053..23f5c3988 100644 --- a/user/event/event_gotls.go +++ b/user/event/event_gotls.go @@ -32,6 +32,8 @@ func (ge *GoTLSEvent) Decode(payload []byte) error { if err = binary.Read(r, binary.LittleEndian, &ge.Data); err != nil { return err } + } else { + ge.Len = 0 } decodedKtime, err := DecodeKtime(int64(ge.TimestampNS), true) if err == nil { diff --git a/user/module/probe_gotls_keylog.go b/user/module/probe_gotls_keylog.go index 3d55a5794..4ff477ef8 100644 --- a/user/module/probe_gotls_keylog.go +++ b/user/module/probe_gotls_keylog.go @@ -22,12 +22,28 @@ import ( manager "github.com/gojue/ebpfmanager" "golang.org/x/sys/unix" "math" + "strings" ) func (g *GoTLSProbe) setupManagersKeylog() error { - - g.logger.Printf("%s\tHOOK type:golang elf, binrayPath:%s\n", g.Name(), g.path) - g.logger.Printf("%s\tHook masterKey function:%s, Address:%x \n", g.Name(), config.GoTlsMasterSecretFunc, g.conf.(*config.GoTLSConfig).GoTlsMasterSecretAddr) + var gotlsConf = g.conf.(*config.GoTLSConfig) + var buildInfo = new(strings.Builder) + for _, setting := range gotlsConf.Buildinfo.Settings { + if setting.Value == "" { + continue + } + buildInfo.WriteString(" ") + buildInfo.WriteString(setting.Key) + buildInfo.WriteString("=") + buildInfo.WriteString(setting.Value) + } + g.logger.Printf("%s\tHOOK type:Golang elf, binrayPath:%s\n", g.Name(), g.path) + g.logger.Printf("%s\tGolang buildInfo version:%s, Params: %s\n", g.Name(), gotlsConf.Buildinfo.GoVersion, buildInfo.String()) + if gotlsConf.IsPieBuildMode { + // buildmode pie is enabled. + g.logger.Printf("%s\tGolang elf buildmode with pie\n", g.Name()) + } + g.logger.Printf("%s\tHook masterKey function:%s, Address:%x \n", g.Name(), config.GoTlsMasterSecretFunc, gotlsConf.GoTlsMasterSecretAddr) var ( sec string diff --git a/user/module/probe_gotls_text.go b/user/module/probe_gotls_text.go index 0a262293a..f1d339882 100644 --- a/user/module/probe_gotls_text.go +++ b/user/module/probe_gotls_text.go @@ -43,6 +43,10 @@ func (g *GoTLSProbe) setupManagersText() error { readFn = "gotls_read_stack" } g.logger.Printf("%s\teBPF Function Name:%s, isRegisterABI:%t\n", g.Name(), fn, g.isRegisterABI) + if g.conf.(*config.GoTLSConfig).IsPieBuildMode { + // buildmode pie is enabled. + g.logger.Printf("%s\tGolang elf buildmode with pie\n", g.Name()) + } g.bpfManager = &manager.Manager{ Probes: []*manager.Probe{ { From c38d3dc7cb742d7fd06c6cc0ea2004dbea5f5e0d Mon Sep 17 00:00:00 2001 From: CFC4N Date: Sat, 30 Mar 2024 18:49:59 +0800 Subject: [PATCH 3/4] fix SA4006: this value of `address` is never used (staticcheck) Signed-off-by: CFC4N --- user/config/config_gotls.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/user/config/config_gotls.go b/user/config/config_gotls.go index 7e9608074..6a56d4098 100644 --- a/user/config/config_gotls.go +++ b/user/config/config_gotls.go @@ -359,17 +359,15 @@ func (gc *GoTLSConfig) findPieSymbolAddr(lfunc string) (uint64, error) { return 0, errors.New("Cant found symbol address on pie model.") } var err error - var address uint64 for _, prog := range gc.goElf.Progs { if prog.Type != elf.PT_LOAD || (prog.Flags&elf.PF_X) == 0 { continue } // For more info on this calculation: stackoverflow.com/a/40249502 - address = f.Value if prog.Vaddr <= f.Value && f.Value < (prog.Vaddr+prog.Memsz) { funcLen := f.End - f.Entry data := make([]byte, funcLen) - address = f.Value - prog.Vaddr + prog.Off + IdaProOffset + address := f.Value - prog.Vaddr + prog.Off + IdaProOffset _, err = prog.ReadAt(data, int64(address)) if err != nil { return 0, fmt.Errorf("search function return: %w", err) From 5b49651d49dd06f3bcbbc9b0bb7673906b6471b2 Mon Sep 17 00:00:00 2001 From: CFC4N Date: Sat, 30 Mar 2024 21:51:05 +0800 Subject: [PATCH 4/4] fix : incorret gotls event payload length. Signed-off-by: CFC4N --- kern/gotls_kern.c | 12 +++++++++--- user/config/config_gotls.go | 1 - user/module/probe_gotls_text.go | 14 ++++++++++++++ 3 files changed, 23 insertions(+), 4 deletions(-) diff --git a/kern/gotls_kern.c b/kern/gotls_kern.c index 0e7a3006c..4b9e108e3 100644 --- a/kern/gotls_kern.c +++ b/kern/gotls_kern.c @@ -106,7 +106,10 @@ static __always_inline int gotls_write(struct pt_regs *ctx, len_ptr = (void *)go_get_argument(ctx, is_register_abi, 4); bpf_probe_read_kernel(&len, sizeof(len), (void *)&len_ptr); - debug_bpf_printk("gotls_write record_type:%d\n", record_type); + if (len == 0) { + return 0; + } + debug_bpf_printk("gotls_write record_type:%d, len:%d\n", record_type, len); if (record_type != recordTypeApplicationData) { return 0; } @@ -115,7 +118,7 @@ static __always_inline int gotls_write(struct pt_regs *ctx, if (!event) { return 0; } - + len = len & 0xFFFF; event->data_len = len; int ret = bpf_probe_read_user(&event->data, sizeof(event->data), (void *)str); @@ -157,7 +160,10 @@ static __always_inline int gotls_read(struct pt_regs *ctx, // Read函数的返回值第一个是int类型，存放在栈里的顺序是5 ret_len_ptr = (void *)go_get_argument_by_stack(ctx, 5); bpf_probe_read_kernel(&ret_len, sizeof(ret_len), (void *)&ret_len_ptr); - if (len == 0) { + if (len <= 0) { + return 0; + } + if (ret_len <= 0 ) { return 0; } diff --git a/user/config/config_gotls.go b/user/config/config_gotls.go index 6a56d4098..a03024ce9 100644 --- a/user/config/config_gotls.go +++ b/user/config/config_gotls.go @@ -182,7 +182,6 @@ func (gc *GoTLSConfig) Check() error { gc.ReadTlsAddrs, err = gc.findRetOffsetsPie(GoTlsReadFunc) if err != nil { - panic(err) return err } } else { diff --git a/user/module/probe_gotls_text.go b/user/module/probe_gotls_text.go index f1d339882..b21513e13 100644 --- a/user/module/probe_gotls_text.go +++ b/user/module/probe_gotls_text.go @@ -23,6 +23,7 @@ import ( manager "github.com/gojue/ebpfmanager" "golang.org/x/sys/unix" "math" + "strings" ) func (g *GoTLSProbe) setupManagersText() error { @@ -42,7 +43,20 @@ func (g *GoTLSProbe) setupManagersText() error { readSec = "uprobe/gotls_read_stack" readFn = "gotls_read_stack" } + var gotlsConf = g.conf.(*config.GoTLSConfig) + var buildInfo = new(strings.Builder) + for _, setting := range gotlsConf.Buildinfo.Settings { + if setting.Value == "" { + continue + } + buildInfo.WriteString(" ") + buildInfo.WriteString(setting.Key) + buildInfo.WriteString("=") + buildInfo.WriteString(setting.Value) + } g.logger.Printf("%s\teBPF Function Name:%s, isRegisterABI:%t\n", g.Name(), fn, g.isRegisterABI) + g.logger.Printf("%s\tGolang buildInfo version:%s, Params: %s\n", g.Name(), gotlsConf.Buildinfo.GoVersion, buildInfo.String()) + if g.conf.(*config.GoTLSConfig).IsPieBuildMode { // buildmode pie is enabled. g.logger.Printf("%s\tGolang elf buildmode with pie\n", g.Name())