Skip to content

Commit

Permalink
Remove memory allocations from chunk parser
Browse files Browse the repository at this point in the history
  • Loading branch information
jerodev committed Jun 25, 2024
1 parent 4d1947b commit 435bdbf
Showing 1 changed file with 11 additions and 16 deletions.
27 changes: 11 additions & 16 deletions main.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@ import (

const (
BUFF_LEN = 2 * 1024 * 1024
LEFTOVER_LEN = 36
ROUTINE_COUNT = 12
)

Expand Down Expand Up @@ -144,30 +143,26 @@ func parseBuffer(tx chan map[string][]int, chunk []byte) {
var temp int
var ok bool

line := make([]byte, 0, LEFTOVER_LEN)
results := map[string][]int{}

var start, ptr int
for _, c := range chunk {
if c == '\n' {
if len(line) > 0 {
if !bytes.ContainsRune(line, ';') {
panic(string(chunk))
}
city, temp = parseLine(chunk[start:ptr])

city, temp = parseLine(line)
if _, ok = results[city]; ok {
results[city] = append(results[city], temp)
} else {
results[city] = []int{temp}
}

if _, ok = results[city]; ok {
results[city] = append(results[city], temp)
} else {
results[city] = []int{temp}
}
ptr++
start = ptr

line = make([]byte, 0, LEFTOVER_LEN)
continue
}
continue
}

line = append(line, c)
ptr++
}

tx <- results
Expand Down

0 comments on commit 435bdbf

Please sign in to comment.