Skip to content

Commit

Permalink
devUpload should not be parsed by egde-view
Browse files Browse the repository at this point in the history 
Since keepSentQueue contains the same or more logs than
devUpload, it's enough to only use those logs when
searching with edge-view.

Signed-off-by: Paul Gaiduk <paulg@zededa.com>
  • Loading branch information
@europaul @eriknordmark
europaul authored and eriknordmark committed Dec 9, 2024
1 parent 4bb28ac commit 2edd016
Show file tree
Hide file tree
Showing 148 changed files with 178,518 additions and 1,728 deletions.
6 changes: 4 additions & 2 deletions pkg/edgeview/go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,9 @@ require (
github.com/fsnotify/fsnotify v1.6.0
github.com/gorilla/websocket v1.5.0
github.com/grandcat/zeroconf v1.0.0
github.com/lf-edge/eve-api/go v0.0.0-20240829123634-7c8ebda876ff
github.com/lf-edge/eve/pkg/pillar v0.0.0-20241011162601-efd973923722
github.com/lf-edge/eve-api/go v0.0.0-20241126174545-53f139cd5166
github.com/lf-edge/eve/pkg/pillar v0.0.0-20241126215939-16062e81d553
github.com/onsi/gomega v1.29.0
github.com/satori/go.uuid v1.2.1-0.20180404165556-75cca531ea76
github.com/shirou/gopsutil v3.21.11+incompatible
github.com/sirupsen/logrus v1.9.3
Expand Down Expand Up @@ -41,4 +42,5 @@ require (
golang.org/x/text v0.14.0 // indirect
google.golang.org/protobuf v1.33.0 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)
18 changes: 14 additions & 4 deletions pkg/edgeview/go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,8 @@ github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4
github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw=
github.com/gabriel-vasile/mimetype v1.4.2 h1:w5qFW6JKBz9Y393Y4q372O9A7cUSequkh1Q7OhCmWKU=
github.com/gabriel-vasile/mimetype v1.4.2/go.mod h1:zApsH/mKG4w07erKIaJPFiX0Tsq9BFQgN3qGY5GnNgA=
github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ=
github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY=
github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s=
Expand All @@ -23,10 +25,14 @@ github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJn
github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY=
github.com/go-playground/validator/v10 v10.15.5 h1:LEBecTWb/1j5TNY1YYG2RcOUN3R7NLylN+x8TTueE24=
github.com/go-playground/validator/v10 v10.15.5/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU=
github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI=
github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls=
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/go-containerregistry v0.14.0 h1:z58vMqHxuwvAsVwvKEkmVBz2TlgBgH5k6koEXBtlYkw=
github.com/google/go-containerregistry v0.14.0/go.mod h1:aiJ2fp/SXvkWgmYHioXnbMdlgB8eXiiYOY55gfN91Wk=
github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38 h1:yAJXTCF9TqKcTiHJAE8dj7HMvPfh66eeA2JYW7eFpSE=
github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc=
github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
github.com/grandcat/zeroconf v1.0.0 h1:uHhahLBKqwWBV6WZUDAT71044vwOTL+McW0mBJvo6kE=
Expand All @@ -37,15 +43,17 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/leodido/go-urn v1.2.4 h1:XlAE/cm/ms7TE/VMVoduSpNBoyc2dOxHs5MZSwAN63Q=
github.com/leodido/go-urn v1.2.4/go.mod h1:7ZrI8mTSeBSHl/UaRyKQW1qZeMgak41ANeCNaVckg+4=
github.com/lf-edge/eve-api/go v0.0.0-20240829123634-7c8ebda876ff h1:3uGTOvWQFQkIrlkFalmzUmXINnzmVOAn5Zx0ryBSzxQ=
github.com/lf-edge/eve-api/go v0.0.0-20240829123634-7c8ebda876ff/go.mod h1:ot6MhAhBXapUDl/hXklaX4kY88T3uC4PTg0D2wD8DzA=
github.com/lf-edge/eve-api/go v0.0.0-20241126174545-53f139cd5166 h1:TDCm5zpp6q3pZz0zO4us3cZFtzm1Tk8JIDqBOioFy1k=
github.com/lf-edge/eve-api/go v0.0.0-20241126174545-53f139cd5166/go.mod h1:ot6MhAhBXapUDl/hXklaX4kY88T3uC4PTg0D2wD8DzA=
github.com/lf-edge/eve/pkg/kube/cnirpc v0.0.0-20240315102754-0f6d1f182e0d h1:tUBb9M6u42LXwHAYHyh22wJeUUQlTpDkXwRXalpRqbo=
github.com/lf-edge/eve/pkg/kube/cnirpc v0.0.0-20240315102754-0f6d1f182e0d/go.mod h1:Nn3juMJJ1G8dyHOebdZyS4jOB/fuxAd5fIajBaWjHr8=
github.com/lf-edge/eve/pkg/pillar v0.0.0-20241011162601-efd973923722 h1:wy0+46dsTULgAqVMmumzT1i2HVSMXSWeT76DuR+Il0M=
github.com/lf-edge/eve/pkg/pillar v0.0.0-20241011162601-efd973923722/go.mod h1:5zB+wlsiTBE2N/Jb0Zr7DaUJMG2OCcAP6X+dyftIdzo=
github.com/lf-edge/eve/pkg/pillar v0.0.0-20241126215939-16062e81d553 h1:89ZRk9plljAhGOkXj1Rgjg0Y0qbllax1QWAQMqUk1q8=
github.com/lf-edge/eve/pkg/pillar v0.0.0-20241126215939-16062e81d553/go.mod h1:1NZojBfxi51+dZrJzeUkTIBUEMlncjNbjnWabPXmIS4=
github.com/miekg/dns v1.1.27/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
github.com/miekg/dns v1.1.43 h1:JKfpVSCB84vrAmHzyrsxB5NAr5kLoMXZArPSw7Qlgyg=
github.com/miekg/dns v1.1.43/go.mod h1:+evo5L0630/F6ca/Z9+GAqzhjGyn8/c+TBaOyfEl0V4=
github.com/onsi/ginkgo/v2 v2.13.0 h1:0jY9lJquiL8fcf3M4LAXN5aMlS/b2BV86HFFPCPMgE4=
github.com/onsi/ginkgo/v2 v2.13.0/go.mod h1:TE309ZR8s5FsKKpuB1YAQYBzCaAfUgatB/xlT/ETL/o=
github.com/onsi/gomega v1.29.0 h1:KIA/t2t5UBzoirT4H9tsML45GEbo3ouUnBHsCfD2tVg=
github.com/onsi/gomega v1.29.0/go.mod h1:9sxs+SwGrKI0+PWe4Fxa9tFQQBG5xSsSbMXOI8PPpoQ=
github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
Expand Down Expand Up @@ -122,6 +130,8 @@ golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk=
golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
golang.org/x/tools v0.18.0 h1:k8NLag8AGHnn+PHbl7g43CtqZAwG60vZkLqgyZgIHgQ=
golang.org/x/tools v0.18.0/go.mod h1:GL7B4CwcLLeo59yx/9UWWuNOW1n3VZ4f5axWfML7Lcg=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI=
google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
Expand Down
35 changes: 35 additions & 0 deletions pkg/edgeview/src/edgeview_test.go
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
// Copyright (c) 2024 Zededa, Inc.
// SPDX-License-Identifier: Apache-2.0

package main

import (
"path"
"testing"
"time"

. "github.com/onsi/gomega"
)

func TestWalkLogDirs(t *testing.T) {
g := NewWithT(t)

// test the walkLogDirs function
newlogDir = "../../newlog/testdata"

const layout = "2006-01-02 15:04:05.000 -0700 MST"
timestamp := "2024-11-13 10:58:52.618 +0100 CET"
parsedTime, err := time.Parse(layout, timestamp)
g.Expect(err).NotTo(HaveOccurred(), "failed to parse timestamp")

from := parsedTime.Add(-1 * time.Second)
to := parsedTime.Add(1 * time.Second)
foundFiles := walkLogDirs(to.Unix(), from.Unix())
g.Expect(foundFiles).To(HaveLen(1), "expected exactly one file to be found")

expected := logfiletime{
filepath: path.Join(newlogDir, "keepSentQueue/dev.log.keep.1731491932618.gz"),
filesec: 1731491932,
}
g.Expect(foundFiles[0]).To(Equal(expected))
}
135 changes: 81 additions & 54 deletions pkg/edgeview/src/log-search.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,13 +10,20 @@ import (
"fmt"
"io"
"os"
"path"
"path/filepath"
"sort"
"strconv"
"strings"
"time"

"github.com/lf-edge/eve-api/go/logs"
"github.com/lf-edge/eve/pkg/pillar/types"
)

var (
newlogDir = "/persist/newlog"
collectDir = newlogDir + "/collect"
)

type logfiletime struct {
Expand Down Expand Up @@ -75,7 +82,7 @@ func runLogSearch(cmds cmdOpt) {
return
}

gfiles := walkLogDirs(t1, t2, now)
gfiles := walkLogDirs(t1, t2)
prog1 := "zcat"
prog2 := "grep"
arg2 := []string{"-E", pattern}
Expand Down Expand Up @@ -106,66 +113,52 @@ func runLogSearch(cmds cmdOpt) {
fmt.Println()
}

func walkLogDirs(t1, t2, now int64) []logfiletime {
func walkLogDirs(toTimestamp, fromTimestamp int64) []logfiletime {
var getfiles []logfiletime

files, err := os.ReadDir("/persist/newlog")
subdirs, err := os.ReadDir(newlogDir)
if err != nil {
fmt.Printf("read /persist/newlog error %v\n", err)
fmt.Printf("read %s error %v\n", newlogDir, err)
return getfiles
}

gzfiles := make(map[string][]string)
for _, dir := range files {
if !dir.IsDir() {
continue
}
if strings.Contains(dir.Name(), "collect") || strings.Contains(dir.Name(), "panic") {
continue
}
if strings.Contains(dir.Name(), "devUpload") && querytype == "app" {
continue
}
if strings.Contains(dir.Name(), "appUpload") && querytype == "dev" {
excludeDirs := []string{"collect", "panic", "devUpload"}
if querytype == "dev" {
excludeDirs = append(excludeDirs, "appUpload")
}
excludeFiles := []string{}
if querytype == "app" {
excludeFiles = append(excludeFiles, "dev")
}
if querytype == "dev" {
excludeFiles = append(excludeFiles, "app")
}

for _, dir := range subdirs {
if filterDir(dir, excludeDirs) {
continue
}
files1, err := os.ReadDir("/persist/newlog/" + dir.Name())

files, err := os.ReadDir(path.Join(newlogDir, dir.Name()))
if err != nil {
fmt.Printf("read %s error %v\n", path.Join(newlogDir, dir.Name()), err)
continue
}
var groupfiles []string
for _, f := range files1 {
info, err := f.Info()
if err != nil {
continue
}
if info.ModTime().Unix() > t1 || info.ModTime().Unix() < t2 {
for _, f := range files {
if filterFile(f, excludeFiles) {
continue
}
groupfiles = append(groupfiles, f.Name())
}
gzfiles["/persist/newlog/"+dir.Name()] = groupfiles
}

for k, g := range gzfiles {
for _, l := range g {
if !strings.Contains(l, "dev") && !strings.Contains(l, "app") {
continue
}
if querytype == "app" && !strings.Contains(l, "app") {
continue
}
if querytype == "dev" && !strings.Contains(l, "dev") {
continue
}
ftime := getFileTime(l)
if ftime == 0 {
timestamp, err := types.GetTimestampFromGzipName(f.Name())
if err != nil {
continue
}
if ftime >= t2 && ftime <= t1 {
file1 := strings.TrimPrefix(l, "./")
ftime := timestamp.Unix() // convert to seconds

if ftime >= fromTimestamp && ftime <= toTimestamp {
file1 := strings.TrimPrefix(f.Name(), "./")
gfile := logfiletime{
filepath: k + "/" + file1,
filepath: path.Join(newlogDir, dir.Name(), file1),
filesec: ftime,
}
getfiles = append(getfiles, gfile)
Expand All @@ -180,26 +173,60 @@ func walkLogDirs(t1, t2, now int64) []logfiletime {
return getfiles
}

func searchLiveLogs(pattern string, now int64, typeStr string, idx *int, logjson bool) {
files, err := os.ReadDir("/persist/newlog/collect")
if err != nil {
fmt.Printf("read /persist/newlog/collect error %v\n", err)
return
func filterDir(dir os.DirEntry, filter []string) bool {
if !dir.IsDir() {
return true
}
for _, name := range filter {
if strings.Contains(dir.Name(), name) {
return true
}
}
return false
}

for _, l := range files {
if !strings.HasPrefix(l.Name(), typeStr) {
continue
func filterFile(file os.DirEntry, filter []string) bool {
if file.IsDir() {
return true
}
for _, name := range filter {
if strings.Contains(file.Name(), name) {
return true
}
file := "/persist/newlog/collect/" + l.Name()
}
return false
}

func searchLiveLogs(pattern string, now int64, typeStr string, idx *int, logjson bool) {
var filesToGrep []string
switch typeStr {
case "dev":
filesToGrep = append(filesToGrep, path.Join(collectDir, "current.device.log"))
case "app":
files, err := os.ReadDir(collectDir)
if err != nil {
fmt.Printf("searchLiveLogs: read %s: error %v\n", collectDir, err)
return
}

for _, l := range files {
if strings.HasPrefix(l.Name(), "app") {
filesToGrep = append(filesToGrep, path.Join(collectDir, l.Name()))
}
}
default:
fmt.Printf("searchLiveLogs: invalid typeStr %v\n", typeStr)
}

for _, file := range filesToGrep {
searchCurrentLogs(pattern, file, typeStr, now, idx, logjson)
}
}

func searchCurrentLogs(pattern, path, typeStr string, now int64, idx *int, logjson bool) {
contents, err := os.ReadFile(path)
if err != nil {
fmt.Printf("read %s file error: %v\n", path, err)
fmt.Printf("searchCurrentLogs: read %s file error: %v\n", path, err)
}
lines := bytes.SplitAfter(contents, []byte("\n"))
var selectlines string
Expand Down
22 changes: 4 additions & 18 deletions pkg/edgeview/src/system.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -160,10 +160,12 @@ func getLogStats() {
isDev = true
}

time1 := getFileTime(l.Name())
if time1 == 0 {
timestamp, err := types.GetTimestampFromGzipName(l.Name())
if err != nil {
continue
}
time1 := timestamp.Unix() // convert to seconds

if isDev && (tmin == 0 || tmin > time1) {
tmin = time1
}
Expand Down Expand Up @@ -308,22 +310,6 @@ func du(currentPath string, info os.FileInfo) int64 {
return size
}

func getFileTime(filename string) int64 {
var fn []string
if strings.Contains(filename, ".gz") && strings.Contains(filename, ".log.") {
fn = strings.Split(filename, ".gz")
}
if len(fn) < 2 {
return 0
}
fn = strings.Split(fn[0], ".log.")
if len(fn) < 2 {
return 0
}
filetime, _ := strconv.Atoi(fn[1])
return int64(filetime / 1000)
}

func getVolume() {
jfiles, err := listJSONFiles("/run/zedagent/AppInstanceConfig")
if err != nil {
Expand Down
Loading

0 comments on commit 2edd016

Please sign in to comment.