Skip to content

Commit

Permalink
[preprocessor/folder] optimize memory usage for hash calculation
Browse files Browse the repository at this point in the history
  • Loading branch information
williamchong committed May 30, 2024
1 parent fe3bdd3 commit d4cf884
Showing 1 changed file with 10 additions and 6 deletions.
16 changes: 10 additions & 6 deletions preprocessor/folder/file.go
Original file line number Diff line number Diff line change
Expand Up @@ -30,19 +30,23 @@ func getFileMetadata(filePath string) (map[string]any, error) {
return nil, err
}

buffer := make([]byte, 512)
n, err := file.Read(buffer)
if err != nil {
return nil, err
}
mediaType := http.DetectContentType(buffer[:n])
file.Seek(0, 0)

Check failure on line 39 in preprocessor/folder/file.go

View workflow job for this annotation

GitHub Actions / lint

Error return value of `file.Seek` is not checked (errcheck)

sha := sha256.New()
md := md5.New()
blake := blake3.New(32, nil)

tee := io.TeeReader(file, sha)
tee = io.TeeReader(tee, md)
tee = io.TeeReader(tee, blake)

bytes, err := io.ReadAll(tee)
writers := io.MultiWriter(sha, md, blake)
_, err = io.Copy(writers, file)
if err != nil {
return nil, err
}
mediaType := http.DetectContentType(bytes)

return map[string]any{
"sha256": hex.EncodeToString(sha.Sum(nil)),
Expand Down

0 comments on commit d4cf884

Please sign in to comment.