-
Notifications
You must be signed in to change notification settings - Fork 128
/
json.go
111 lines (92 loc) · 2.25 KB
/
json.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
package main
import (
"bufio"
"encoding/json"
"fmt"
"io"
"os"
)
// Try to JSON decode the bytes
func tryUnmarshal(b []byte) error {
var v interface{}
err := json.Unmarshal(b, &v)
return err
}
//Copy JSON Rows and return list of errors
func copyJSONRows(i *Import, reader *bufio.Reader, ignoreErrors bool) (error, int, int) {
success := 0
failed := 0
for {
// ReadBytes instead of a Scanner because it can deal with very long lines
// which happens often with big JSON objects
line, err := reader.ReadBytes('\n')
if err == io.EOF {
err = nil
break
}
if err != nil {
err = fmt.Errorf("%s: %s", err, line)
return err, success, failed
}
err = tryUnmarshal(line)
if err != nil {
failed++
if ignoreErrors {
os.Stderr.WriteString(string(line))
continue
} else {
err = fmt.Errorf("%s: %s", err, line)
return err, success, failed
}
}
err = i.AddRow(string(line))
if err != nil {
failed++
if ignoreErrors {
os.Stderr.WriteString(string(line))
continue
} else {
err = fmt.Errorf("%s: %s", err, line)
return err, success, failed
}
}
success++
}
return nil, success, failed
}
func importJSON(filename string, connStr string, schema string, tableName string, ignoreErrors bool, dataType string) error {
db, err := connect(connStr, schema)
if err != nil {
return err
}
defer db.Close()
i, err := NewJSONImport(db, schema, tableName, "data", dataType)
if err != nil {
return err
}
var success, failed int
if filename == "" {
reader := bufio.NewReader(os.Stdin)
err, success, failed = copyJSONRows(i, reader, ignoreErrors)
} else {
file, err := os.Open(filename)
if err != nil {
return err
}
defer file.Close()
bar := NewProgressBar(file)
reader := bufio.NewReader(io.TeeReader(file, bar))
bar.Start()
err, success, failed = copyJSONRows(i, reader, ignoreErrors)
bar.Finish()
}
if err != nil {
lineNumber := success + failed
return fmt.Errorf("line %d: %s", lineNumber, err)
}
fmt.Println(fmt.Sprintf("%d rows imported into %s.%s", success, schema, tableName))
if ignoreErrors && failed > 0 {
fmt.Println(fmt.Sprintf("%d rows could not be imported into %s.%s and have been written to stderr.", failed, schema, tableName))
}
return i.Commit()
}