Skip to content

Commit

Permalink
Strip "unsafe" from ndt5 and ndt7 UUIDs (#1107)
Browse files Browse the repository at this point in the history 
* Strip _unsafe from ndt5 UUIDs
* Strip _unsafe from ndt7 UUIDs
* Add new unsafe test data
  • Loading branch information
@stephen-soltesz
stephen-soltesz authored Dec 1, 2022
1 parent 1700f84 commit 163dadf
Show file tree
Hide file tree
Showing 6 changed files with 67 additions and 18 deletions.
2 changes: 2 additions & 0 deletions parser/ndt5_result.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -162,6 +162,7 @@ func (dp *NDT5ResultParser) newResult(test []byte, parser schema.ParseInfo, date
func (dp *NDT5ResultParser) prepareS2CRow(row *schema.NDT5ResultRowV2) {
// Record S2C result.
s2c := row.Raw.S2C
s2c.UUID = strings.ReplaceAll(s2c.UUID, "_unsafe", "")
row.ID = s2c.UUID
row.A = &schema.NDT5Summary{
UUID: s2c.UUID,
Expand Down Expand Up @@ -191,6 +192,7 @@ func (dp *NDT5ResultParser) prepareS2CRow(row *schema.NDT5ResultRowV2) {
func (dp *NDT5ResultParser) prepareC2SRow(row *schema.NDT5ResultRowV2) {
// Record C2S result.
c2s := row.Raw.C2S
c2s.UUID = strings.ReplaceAll(c2s.UUID, "_unsafe", "")
row.ID = c2s.UUID
row.A = &schema.NDT5Summary{
UUID: c2s.UUID,
Expand Down
10 changes: 10 additions & 0 deletions parser/ndt5_result_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,10 @@ func TestNDT5ResultParser_ParseAndInsert(t *testing.T) {
testName: `ndt-m9pcq_1652405655_000000000014FD22.json`,
expectTCPInfo: true,
},
{
name: "success-remove-download-unsafe-uuid",
testName: `ndt-rczlq_1666977535_unsafe_0000000000169592.json`,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
Expand Down Expand Up @@ -91,6 +95,9 @@ func TestNDT5ResultParser_ParseAndInsert(t *testing.T) {
download.Raw.Control.ClientMetadata[0].Name,
download.Raw.Control.ClientMetadata[0].Value)
}
if strings.Contains(download.ID, "_unsafe") || strings.Contains(download.A.UUID, "_unsafe") {
t.Fatalf("ID or A.UUID contain the string '_unsafe'")
}
if download.Raw.S2C == nil {
t.Fatalf("Raw.S2C is nil")
}
Expand Down Expand Up @@ -123,6 +130,9 @@ func TestNDT5ResultParser_ParseAndInsert(t *testing.T) {
upload.Raw.Control.ClientMetadata[0].Name,
upload.Raw.Control.ClientMetadata[0].Value)
}
if strings.Contains(upload.ID, "_unsafe") || strings.Contains(upload.A.UUID, "_unsafe") {
t.Fatalf("ID or A.UUID contain the string '_unsafe'")
}
if upload.Raw.C2S == nil {
t.Fatalf("Raw.C2S is nil")
}
Expand Down
2 changes: 2 additions & 0 deletions parser/ndt7_result.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -132,6 +132,7 @@ func (dp *NDT7ResultParser) ParseAndInsert(meta map[string]bigquery.Value, testN
}

func downSummary(down *model.ArchivalData) schema.NDT7Summary {
down.UUID = strings.ReplaceAll(down.UUID, "_unsafe", "")
return schema.NDT7Summary{
UUID: down.UUID,
TestTime: down.StartTime,
Expand All @@ -142,6 +143,7 @@ func downSummary(down *model.ArchivalData) schema.NDT7Summary {
}
}
func upSummary(up *model.ArchivalData) schema.NDT7Summary {
up.UUID = strings.ReplaceAll(up.UUID, "_unsafe", "")
return schema.NDT7Summary{
UUID: up.UUID,
TestTime: up.StartTime,
Expand Down
69 changes: 51 additions & 18 deletions parser/ndt7_result_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package parser_test

import (
"io/ioutil"
"path"
"strings"
"testing"

Expand All @@ -14,6 +15,30 @@ import (
"github.com/m-lab/go/pretty"
)

func setupNDT7InMemoryParser(t *testing.T, testName string) (*schema.NDT7ResultRow, error) {
ins := newInMemorySink()
n := parser.NewNDT7ResultParser(ins, "test", "_suffix")

resultData, err := ioutil.ReadFile(path.Join("testdata/NDT7Result/", testName))
if err != nil {
t.Fatalf(err.Error())
}
meta := map[string]bigquery.Value{
"filename": "gs://mlab-test-bucket/ndt/ndt7/2020/03/18/ndt_ndt7_2020_03_18_20200318T003853.425987Z-ndt7-mlab3-syd03-ndt.tgz",
"date": civil.Date{Year: 2020, Month: 3, Day: 18},
}
err = n.ParseAndInsert(meta, testName, resultData)
if err != nil {
return nil, err
}
if n.Accepted() != 1 {
t.Fatal("Failed to insert snaplog data.", ins)
}
n.Flush()
row := ins.data[0].(*schema.NDT7ResultRow)
return row, err
}

func TestNDT7ResultParser_ParseAndInsert(t *testing.T) {
tests := []struct {
name string
Expand All @@ -31,26 +56,10 @@ func TestNDT7ResultParser_ParseAndInsert(t *testing.T) {
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
ins := newInMemorySink()
n := parser.NewNDT7ResultParser(ins, "test", "_suffix")

resultData, err := ioutil.ReadFile(`testdata/NDT7Result/` + tt.testName)
if err != nil {
t.Fatalf(err.Error())
}
meta := map[string]bigquery.Value{
"filename": "gs://mlab-test-bucket/ndt/ndt7/2020/03/18/ndt_ndt7_2020_03_18_20200318T003853.425987Z-ndt7-mlab3-syd03-ndt.tgz",
"date": civil.Date{Year: 2020, Month: 3, Day: 18},
}

if err := n.ParseAndInsert(meta, tt.testName, resultData); (err != nil) != tt.wantErr {
row, err := setupNDT7InMemoryParser(t, tt.testName)
if (err != nil) != tt.wantErr {
t.Errorf("NDT7ResultParser.ParseAndInsert() error = %v, wantErr %v", err, tt.wantErr)
}
if n.Accepted() != 1 {
t.Fatal("Failed to insert snaplog data.", ins)
}
n.Flush()
row := ins.data[0].(*schema.NDT7ResultRow)
if row.Raw.Download != nil {
exp := schema.NDT7Summary{
UUID: "ndt-knwp4_1583603744_000000000000590E",
Expand Down Expand Up @@ -136,6 +145,30 @@ func TestNDT7ResultParser_ParseAndInsert(t *testing.T) {
}
}

func TestNDT7ResultParser_ParseAndInsertUnsafe(t *testing.T) {
tests := []struct {
name string
testName string
wantErr bool
}{
{
name: "success-remove-unsafe-uuid",
testName: `ndt7-download-20221130T230746.606388371Z.ndt-rczlq_1666977535_unsafe_000000000016912D.json`,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
row, err := setupNDT7InMemoryParser(t, tt.testName)
if (err != nil) != tt.wantErr {
t.Errorf("NDT7ResultParser.ParseAndInsert() error = %v, wantErr %v", err, tt.wantErr)
}
if strings.Contains(row.ID, "_unsafe") || strings.Contains(row.A.UUID, "_unsafe") || strings.Contains(row.Raw.Download.UUID, "_unsafe") {
t.Fatalf("ID or A.UUID contain the string '_unsafe'")
}
})
}
}

func TestNDT7ResultParser_IsParsable(t *testing.T) {
tests := []struct {
name string
Expand Down
1 change: 1 addition & 0 deletions parser/testdata/NDT5Result/ndt-rczlq_1666977535_unsafe_0000000000169592.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{"GitShortCommit":"65ed3f0","Version":"v0.20.17","ServerIP":"212.73.231.203","ServerPort":3001,"ClientIP":"93.26.109.189","ClientPort":35105,"StartTime":"2022-11-30T23:46:06.03304386Z","EndTime":"2022-11-30T23:46:16.207358103Z","Control":{"UUID":"ndt-rczlq_1666977535_unsafe_0000000000169592","Protocol":"PLAIN","MessageProtocol":"JSON","ServerMetadata":[{"Name":"deployment","Value":"stable"},{"Name":"type","Value":"physical"}]},"C2S":{"ServerIP":"212.73.231.203","ServerPort":36081,"ClientIP":"193.248.160.8","ClientPort":60684,"UUID":"ndt-rczlq_1666977535_unsafe_0000000000168F69","StartTime":"2022-11-30T22:52:46.910235486Z","EndTime":"2022-11-30T22:52:56.910490315Z","MeanThroughputMbps":14.133278312335156},"S2C":{"UUID":"ndt-rczlq_1666977535_unsafe_0000000000169594","ServerIP":"212.73.231.203","ServerPort":36245,"ClientIP":"93.26.109.189","ClientPort":35137,"StartTime":"2022-11-30T23:46:06.080155486Z","EndTime":"2022-11-30T23:46:16.10196625Z","MeanThroughputMbps":218.32357147346343,"MinRTT":12000000,"MaxRTT":41000000,"SumRTT":2596000000,"CountRTT":101,"ClientReportedMbps":219.151,"TCPInfo":{"State":1,"CAState":0,"Retransmits":0,"Probes":0,"Backoff":0,"Options":7,"WScale":119,"AppLimited":0,"RTO":232000,"ATO":0,"SndMSS":1448,"RcvMSS":536,"Unacked":578,"Sacked":0,"Lost":0,"Retrans":0,"Fackets":0,"LastDataSent":0,"LastAckSent":0,"LastDataRecv":10000,"LastAckRecv":0,"PMTU":1500,"RcvSsThresh":64076,"RTT":30444,"RTTVar":455,"SndSsThresh":426,"SndCwnd":591,"AdvMSS":1448,"Reordering":3,"RcvRTT":0,"RcvSpace":14600,"TotalRetrans":224,"PacingRate":33730939,"MaxPacingRate":-1,"BytesAcked":273499688,"BytesReceived":0,"SegsOut":189770,"SegsIn":79124,"NotsentBytes":3160984,"MinRTT":11817,"DataSegsIn":0,"DataSegsOut":189770,"DeliveryRate":26132932,"BusyTime":10000000,"RWndLimited":424000,"SndBufLimited":0,"Delivered":188969,"DeliveredCE":0,"BytesSent":274660984,"BytesRetrans":324352,"DSackDups":0,"ReordSeen":0,"RcvOooPack":0,"SndWnd":3009280}}}
1 change: 1 addition & 0 deletions ...dt7-download-20221130T230746.606388371Z.ndt-rczlq_1666977535_unsafe_000000000016912D.json
View file

Large diffs are not rendered by default.

0 comments on commit 163dadf

Please sign in to comment.