From df18476a3af44f299614a3c18a985bc4240c7142 Mon Sep 17 00:00:00 2001 From: Vo Duy Khanh <39976007+josh-vo@users.noreply.github.com> Date: Wed, 26 Feb 2020 13:40:12 +0700 Subject: [PATCH 01/17] Update version Go from 1.13.3 to 1.13.7 --- .travis.yml | 2 +- README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index ec3064f..937f1e5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,6 @@ language: go go: - - 1.13.3 + - 1.13.7 env: "PATH=/home/travis/gopath/bin:$PATH" before_install: - go get github.com/mattn/gom diff --git a/README.md b/README.md index 58d0789..2955277 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ cli-kintone is a command line utility for exporting and importing kintone App da ### Requirement -- Go 1.13.3 +- Go 1.13.7 - Git and Mercurial to be able to clone the packages ### Mac OS X/Linux From 8251b202f7bc75c14a7da9078e9c79d106794234 Mon Sep 17 00:00:00 2001 From: Vo Duy Khanh <39976007+josh-vo@users.noreply.github.com> Date: Wed, 26 Feb 2020 18:02:26 +0700 Subject: [PATCH 02/17] Adding function to remove BOM character. --- import-with-bulkRequest.go | 4 ++-- utils.go | 20 ++++++++++++++++++++ 2 files changed, 22 insertions(+), 2 deletions(-) create mode 100644 utils.go diff --git a/import-with-bulkRequest.go b/import-with-bulkRequest.go index e72d7a2..ff40b55 100644 --- a/import-with-bulkRequest.go +++ b/import-with-bulkRequest.go @@ -106,8 +106,8 @@ func addSubField(app *kintone.App, column *Column, col string, table *SubRecord) return nil } func importFromCSV(app *kintone.App, _reader io.Reader) error { - - reader := csv.NewReader(getReader(_reader)) + readerWithoutBOM := removeBOMCharacter(_reader) + reader := csv.NewReader(getReader(readerWithoutBOM)) head := true var columns Columns diff --git a/utils.go b/utils.go new file mode 100644 index 0000000..1b82902 --- /dev/null +++ b/utils.go @@ -0,0 +1,20 @@ +package main + +import ( + "bufio" + "log" + "io" +) + +func removeBOMCharacter(reader io.Reader) io.Reader { + bufferReader := bufio.NewReader(reader) + r, _, err := bufferReader.ReadRune() + if err != nil { + log.Fatal(err) + } + + if r != '\uFEFF' { + bufferReader.UnreadRune() // Not a BOM -- put the rune back + } + return bufferReader +} From 1054c16abe077db6285ecea69bdbbda151324b0b Mon Sep 17 00:00:00 2001 From: minhhy2801 Date: Thu, 27 Feb 2020 08:37:33 +0700 Subject: [PATCH 03/17] Add GBK and big5 encoding support for Chinese characters. --- main.go | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/main.go b/main.go index e8708a2..4b6260b 100644 --- a/main.go +++ b/main.go @@ -9,9 +9,11 @@ import ( "github.com/howeyc/gopass" "github.com/kintone/go-kintone" - "golang.org/x/text/encoding" - "golang.org/x/text/encoding/japanese" - "golang.org/x/text/encoding/unicode" + "golang.org/x/text/encoding" + "golang.org/x/text/encoding/unicode" + "golang.org/x/text/encoding/japanese" + "golang.org/x/text/encoding/simplifiedchinese" + "golang.org/x/text/encoding/traditionalchinese" flags "github.com/jessevdk/go-flags" ) @@ -195,7 +197,11 @@ func getEncoding() encoding.Encoding { case "euc-jp": return japanese.EUCJP case "sjis": - return japanese.ShiftJIS + return japanese.ShiftJIS + case "gbk": + return simplifiedchinese.GBK + case "big5": + return traditionalchinese.Big5 default: return nil } From 8132fa5e16bb66ec671bddecc1695a2187301688 Mon Sep 17 00:00:00 2001 From: minhhy2801 Date: Thu, 27 Feb 2020 08:46:59 +0700 Subject: [PATCH 04/17] format style space --- main.go | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/main.go b/main.go index 4b6260b..02f59a3 100644 --- a/main.go +++ b/main.go @@ -9,11 +9,11 @@ import ( "github.com/howeyc/gopass" "github.com/kintone/go-kintone" - "golang.org/x/text/encoding" - "golang.org/x/text/encoding/unicode" - "golang.org/x/text/encoding/japanese" - "golang.org/x/text/encoding/simplifiedchinese" - "golang.org/x/text/encoding/traditionalchinese" + "golang.org/x/text/encoding" + "golang.org/x/text/encoding/unicode" + "golang.org/x/text/encoding/japanese" + "golang.org/x/text/encoding/simplifiedchinese" + "golang.org/x/text/encoding/traditionalchinese" flags "github.com/jessevdk/go-flags" ) @@ -197,11 +197,11 @@ func getEncoding() encoding.Encoding { case "euc-jp": return japanese.EUCJP case "sjis": - return japanese.ShiftJIS - case "gbk": - return simplifiedchinese.GBK - case "big5": - return traditionalchinese.Big5 + return japanese.ShiftJIS + case "gbk": + return simplifiedchinese.GBK + case "big5": + return traditionalchinese.Big5 default: return nil } From 710781707f027b07c209c7788465458d20ed924a Mon Sep 17 00:00:00 2001 From: minhhy2801 Date: Thu, 27 Feb 2020 08:49:29 +0700 Subject: [PATCH 05/17] format style Tabs --- main.go | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/main.go b/main.go index 02f59a3..68bcc9e 100644 --- a/main.go +++ b/main.go @@ -9,11 +9,11 @@ import ( "github.com/howeyc/gopass" "github.com/kintone/go-kintone" - "golang.org/x/text/encoding" - "golang.org/x/text/encoding/unicode" - "golang.org/x/text/encoding/japanese" - "golang.org/x/text/encoding/simplifiedchinese" - "golang.org/x/text/encoding/traditionalchinese" + "golang.org/x/text/encoding" + "golang.org/x/text/encoding/unicode" + "golang.org/x/text/encoding/japanese" + "golang.org/x/text/encoding/simplifiedchinese" + "golang.org/x/text/encoding/traditionalchinese" flags "github.com/jessevdk/go-flags" ) @@ -197,11 +197,11 @@ func getEncoding() encoding.Encoding { case "euc-jp": return japanese.EUCJP case "sjis": - return japanese.ShiftJIS - case "gbk": - return simplifiedchinese.GBK - case "big5": - return traditionalchinese.Big5 + return japanese.ShiftJIS + case "gbk": + return simplifiedchinese.GBK + case "big5": + return traditionalchinese.Big5 default: return nil } From 1accc3325d310a9d480c5cdf70731256425f6a34 Mon Sep 17 00:00:00 2001 From: Vo Duy Khanh <39976007+josh-vo@users.noreply.github.com> Date: Thu, 27 Feb 2020 09:17:21 +0700 Subject: [PATCH 06/17] Update removeBOMCharacter function into getReader function --- import-with-bulkRequest.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/import-with-bulkRequest.go b/import-with-bulkRequest.go index ff40b55..d91703f 100644 --- a/import-with-bulkRequest.go +++ b/import-with-bulkRequest.go @@ -23,11 +23,13 @@ type SubRecord struct { } func getReader(reader io.Reader) io.Reader { + readerWithoutBOM := removeBOMCharacter(reader) + encoding := getEncoding() if encoding == nil { - return reader + return readerWithoutBOM } - return transform.NewReader(reader, encoding.NewDecoder()) + return transform.NewReader(readerWithoutBOM, encoding.NewDecoder()) } // delete specific records From f4215d911574a70af7123f4082dd76ef88c64e37 Mon Sep 17 00:00:00 2001 From: Vo Duy Khanh <39976007+josh-vo@users.noreply.github.com> Date: Thu, 27 Feb 2020 09:19:11 +0700 Subject: [PATCH 07/17] Remove check BOM at importFromCSV function --- import-with-bulkRequest.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/import-with-bulkRequest.go b/import-with-bulkRequest.go index d91703f..9f27274 100644 --- a/import-with-bulkRequest.go +++ b/import-with-bulkRequest.go @@ -108,8 +108,7 @@ func addSubField(app *kintone.App, column *Column, col string, table *SubRecord) return nil } func importFromCSV(app *kintone.App, _reader io.Reader) error { - readerWithoutBOM := removeBOMCharacter(_reader) - reader := csv.NewReader(getReader(readerWithoutBOM)) + reader := csv.NewReader(getReader(_reader)) head := true var columns Columns From c06d868b9d6e91544e56b1f05fbbb63adec1f538 Mon Sep 17 00:00:00 2001 From: Vo Duy Khanh <39976007+josh-vo@users.noreply.github.com> Date: Thu, 27 Feb 2020 09:21:10 +0700 Subject: [PATCH 08/17] Revert format at importFromCSV --- import-with-bulkRequest.go | 1 + 1 file changed, 1 insertion(+) diff --git a/import-with-bulkRequest.go b/import-with-bulkRequest.go index 9f27274..67782cb 100644 --- a/import-with-bulkRequest.go +++ b/import-with-bulkRequest.go @@ -108,6 +108,7 @@ func addSubField(app *kintone.App, column *Column, col string, table *SubRecord) return nil } func importFromCSV(app *kintone.App, _reader io.Reader) error { + reader := csv.NewReader(getReader(_reader)) head := true From 8e8cd2d42c365bf4181c8798acc68a2ecd68d09a Mon Sep 17 00:00:00 2001 From: Vo Duy Khanh <39976007+josh-vo@users.noreply.github.com> Date: Thu, 27 Feb 2020 09:48:26 +0700 Subject: [PATCH 09/17] Reformat space to tab ultis.go --- utils.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/utils.go b/utils.go index 1b82902..bdfa3b8 100644 --- a/utils.go +++ b/utils.go @@ -8,13 +8,13 @@ import ( func removeBOMCharacter(reader io.Reader) io.Reader { bufferReader := bufio.NewReader(reader) - r, _, err := bufferReader.ReadRune() - if err != nil { - log.Fatal(err) + r, _, err := bufferReader.ReadRune() + if err != nil { + log.Fatal(err) + } + + if r != '\uFEFF' { + bufferReader.UnreadRune() // Not a BOM -- put the rune back } - - if r != '\uFEFF' { - bufferReader.UnreadRune() // Not a BOM -- put the rune back - } return bufferReader } From dee8c4c389bfe316bd65e75a18679fb3644c43b6 Mon Sep 17 00:00:00 2001 From: minhhy2801 Date: Mon, 2 Mar 2020 09:49:02 +0700 Subject: [PATCH 10/17] update readme 'gbk' or 'big5' --- README.md | 2 +- main.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 58d0789..b233fbb 100644 --- a/README.md +++ b/README.md @@ -103,7 +103,7 @@ https://github.com/kintone/cli-kintone/releases -o= Output format. Specify either 'json' or 'csv' (default: csv) -e= Character encoding (default: utf-8). Only support the encoding below both field code and data itself: - 'utf-8', 'utf-16', 'utf-16be-with-signature', 'utf-16le-with-signature', 'sjis' or'euc-jp' + 'utf-8', 'utf-16', 'utf-16be-with-signature', 'utf-16le-with-signature', 'sjis' or'euc-jp', 'gbk' or 'big5' -U= Basic authentication user name -P= Basic authentication password -q= Query string diff --git a/main.go b/main.go index 68bcc9e..804db15 100644 --- a/main.go +++ b/main.go @@ -39,7 +39,7 @@ type Configure struct { APIToken string `short:"t" default:"" description:"API token"` GuestSpaceID uint64 `short:"g" default:"0" description:"Guest Space ID"` Format string `short:"o" default:"csv" description:"Output format. Specify either 'json' or 'csv'"` - Encoding string `short:"e" default:"utf-8" description:"Character encoding (default: utf-8).\n Only support the encoding below both field code and data itself: \n 'utf-8', 'utf-16', 'utf-16be-with-signature', 'utf-16le-with-signature', 'sjis' or 'euc-jp'"` + Encoding string `short:"e" default:"utf-8" description:"Character encoding (default: utf-8).\n Only support the encoding below both field code and data itself: \n 'utf-8', 'utf-16', 'utf-16be-with-signature', 'utf-16le-with-signature', 'sjis' or 'euc-jp', 'gbk' or 'big5'"` BasicAuthUser string `short:"U" default:"" description:"Basic authentication user name"` BasicAuthPassword string `short:"P" default:"" description:"Basic authentication password"` Query string `short:"q" default:"" description:"Query string"` From 288e7dd48c8d2b98b3e9f61ac210c08b66fd4fbe Mon Sep 17 00:00:00 2001 From: minhhy2801 Date: Thu, 5 Mar 2020 08:54:03 +0700 Subject: [PATCH 11/17] Remove unused functions/methods --- main.go | 25 ------------------------- 1 file changed, 25 deletions(-) diff --git a/main.go b/main.go index e8708a2..80a8321 100644 --- a/main.go +++ b/main.go @@ -77,31 +77,6 @@ type Cell struct { // Row config type Row []*Cell -func (p Columns) Len() int { - return len(p) -} - -func (p Columns) Swap(i, j int) { - p[i], p[j] = p[j], p[i] -} - -func (p Columns) Less(i, j int) bool { - p1 := p[i] - code1 := p1.Code - if p1.IsSubField { - code1 = p1.Table - } - p2 := p[j] - code2 := p2.Code - if p2.IsSubField { - code2 = p2.Table - } - if code1 == code2 { - return p[i].Code < p[j].Code - } - return code1 < code2 -} - func getFields(app *kintone.App) (map[string]*kintone.FieldInfo, error) { fields, err := app.Fields() if err != nil { From 18553de3360692ed64562cf58282b29aab94fdfb Mon Sep 17 00:00:00 2001 From: minhhy2801 Date: Mon, 9 Mar 2020 15:33:13 +0700 Subject: [PATCH 12/17] Reorder the --import/--export options on the guide/help --- main.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/main.go b/main.go index e8708a2..972d264 100644 --- a/main.go +++ b/main.go @@ -30,6 +30,8 @@ const EXPORT_ROW_LIMIT = 500 // Configure of this package type Configure struct { + IsImport bool `long:"import" description:"Import data from stdin. If \"-f\" is also specified, data is imported from the file instead"` + IsExport bool `long:"export" description:"Export kintone data to stdout"` Domain string `short:"d" default:"" description:"Domain name (specify the FQDN)"` AppID uint64 `short:"a" default:"0" description:"App ID"` Login string `short:"u" default:"" description:"User's log in name"` @@ -46,8 +48,6 @@ type Configure struct { FileDir string `short:"b" default:"" description:"Attachment file directory"` DeleteAll bool `short:"D" description:"Delete records before insert. You can specify the deleting record condition by option \"-q\""` Line uint64 `short:"l" default:"1" description:"Position index of data in the input file"` - IsImport bool `long:"import" description:"Import data from stdin. If \"-f\" is also specified, data is imported from the file instead"` - IsExport bool `long:"export" description:"Export kintone data to stdout"` Version bool `short:"v" long:"version" description:"Version of cli-kintone"` } From e153e4c1855301e5f9bb96e323b2e4e7169a4d47 Mon Sep 17 00:00:00 2001 From: minhhy2801 Date: Mon, 9 Mar 2020 15:33:27 +0700 Subject: [PATCH 13/17] Reorder guide for user and developer --- README.md | 87 +++++++++---------------------------------------------- 1 file changed, 14 insertions(+), 73 deletions(-) diff --git a/README.md b/README.md index 58d0789..0fe6e11 100644 --- a/README.md +++ b/README.md @@ -7,76 +7,6 @@ cli-kintone is a command line utility for exporting and importing kintone App da 0.10.2 -## How to Build - -### Requirement - -- Go 1.13.3 -- Git and Mercurial to be able to clone the packages - -### Mac OS X/Linux -#### Step 1: Creating folder to develop -``` -mkdir -p /tmp/dev-cli-kintone/src -``` -Note: "/tmp/dev-cli-kintone" is the path to project at local, can be changed to match with the project at local of you. - -#### Step 2: Creating variable environment GOPATH - -``` -export GOPATH=/tmp/dev-cli-kintone -``` - -#### Step 3: Getting cli-kintone repository -``` -cd ${GOPATH}/src -git clone https://github.com/kintone/cli-kintone.git -``` - -#### Step 4: Install dependencies -``` -cd ${GOPATH}/src/cli-kintone -go get github.com/mattn/gom -sudo ln -s $GOPATH/bin/gom /usr/local/bin/gom # Link package gom to directory "/usr/local/" to use globally -gom -production install -``` - -#### Step 5: Build -``` -mv vendor/ src -gom build -``` - -### Windows -#### Step 1: Creating folder to develop -``` -mkdir -p c:\tmp\dev-cli-kintone\src -``` -Note: "c:\tmp\dev-cli-kintone" is the path to project at local, can be changed to match with the project at local of you. - -#### Step 2: Creating variable environment GOPATH - -``` -set GOPATH=c:\tmp\dev-cli-kintone -``` - -#### Step 3: Getting cli-kintone repository -``` -cd %GOPATH%\src -git clone https://github.com/kintone/cli-kintone.git -``` - -#### Step 4: Install dependencies -``` -cd %GOPATH%\src\cli-kintone -go get github.com/mattn/gom -..\..\bin\gom.exe -production install -``` - -#### Step 5: Build -``` -..\..\bin\gom.exe build -``` ## Downloads @@ -166,9 +96,6 @@ cli-kintone --import -a -d -t -f -l 25 ``` printf "name,age\nJohn,37\nJane,29" | cli-kintone --import -a -d -t ``` -## Documents for Basic Usage -English: https://developer.kintone.io/hc/en-us/articles/115002614853 -Japanese: https://developer.cybozu.io/hc/ja/articles/202957070 ## Restriction * The limit of the file upload size is 10 MB. @@ -176,6 +103,20 @@ Japanese: https://developer.cybozu.io/hc/ja/articles/202957070 * The following record data cannot be retrieved: Category, Status, Field group. * The following fields cannot be retrieved if they are set inside a Field group: Record number, Created by, Created datetime, Updated by, Updated datetime, Blank space, Label, Border. +## Documents for Basic Usage +English: https://developer.kintone.io/hc/en-us/articles/115002614853 +Japanese: https://developer.cybozu.io/hc/ja/articles/202957070 + +## How to Build + +### Requirement + +- Go 1.13.3 +- Git and Mercurial to be able to clone the packages + +### [Mac OS X/Linux](./docs/BuildForMacLinux.md) + +### [Windows](./docs/BuildForWindows.md) ## License GPL v2 From f537c7627c47a3d6039fbabda93045f20ea5ab5a Mon Sep 17 00:00:00 2001 From: minhhy2801 Date: Mon, 9 Mar 2020 15:33:39 +0700 Subject: [PATCH 14/17] Separate [How to Build > Mac OS X/Linux] and [How to Build > Windows] to child pages --- docs/BuildForMacLinux.md | 36 ++++++++++++++++++++++++++++++++++++ docs/BuildForWindows.md | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 70 insertions(+) create mode 100644 docs/BuildForMacLinux.md create mode 100644 docs/BuildForWindows.md diff --git a/docs/BuildForMacLinux.md b/docs/BuildForMacLinux.md new file mode 100644 index 0000000..0e63f5e --- /dev/null +++ b/docs/BuildForMacLinux.md @@ -0,0 +1,36 @@ +## How to Build Mac OS X/Linux +#### Step 1: Creating folder to develop +``` +mkdir -p /tmp/dev-cli-kintone/src +``` +Note: "/tmp/dev-cli-kintone" is the path to project at local, can be changed to match with the project at local of you. + +#### Step 2: Creating variable environment GOPATH + +``` +export GOPATH=/tmp/dev-cli-kintone +``` + +#### Step 3: Getting cli-kintone repository +``` +cd ${GOPATH}/src +git clone https://github.com/kintone/cli-kintone.git +``` + +#### Step 4: Install dependencies +``` +cd ${GOPATH}/src/cli-kintone +go get github.com/mattn/gom +sudo ln -s $GOPATH/bin/gom /usr/local/bin/gom # Link package gom to directory "/usr/local/" to use globally +gom -production install +``` + +#### Step 5: Build +``` +mv vendor/ src +gom build +``` + +## Copyright + +Copyright(c) Cybozu, Inc. diff --git a/docs/BuildForWindows.md b/docs/BuildForWindows.md new file mode 100644 index 0000000..32cf7ea --- /dev/null +++ b/docs/BuildForWindows.md @@ -0,0 +1,34 @@ +## How to Build Windows +#### Step 1: Creating folder to develop +``` +mkdir -p c:\tmp\dev-cli-kintone\src +``` +Note: "c:\tmp\dev-cli-kintone" is the path to project at local, can be changed to match with the project at local of you. + +#### Step 2: Creating variable environment GOPATH + +``` +set GOPATH=c:\tmp\dev-cli-kintone +``` + +#### Step 3: Getting cli-kintone repository +``` +cd %GOPATH%\src +git clone https://github.com/kintone/cli-kintone.git +``` + +#### Step 4: Install dependencies +``` +cd %GOPATH%\src\cli-kintone +go get github.com/mattn/gom +..\..\bin\gom.exe -production install +``` + +#### Step 5: Build +``` +..\..\bin\gom.exe build +``` + +## Copyright + +Copyright(c) Cybozu, Inc. From 42bf21906eef99ea0816ccfae015af59d9a202e5 Mon Sep 17 00:00:00 2001 From: minhhy2801 Date: Mon, 9 Mar 2020 16:03:02 +0700 Subject: [PATCH 15/17] update readme --- README.md | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 0fe6e11..a589d15 100644 --- a/README.md +++ b/README.md @@ -24,6 +24,8 @@ https://github.com/kintone/cli-kintone/releases cli-kintone [OPTIONS] Application Options: + --import Import data from stdin. If "-f" is also specified, data is imported from the file instead + --export Export kintone data to stdout -d= Domain name (specify the FQDN) -a= App ID (default: 0) -u= User's log in name @@ -42,8 +44,6 @@ https://github.com/kintone/cli-kintone/releases -b= Attachment file directory -D Delete records before insert. You can specify the deleting record condition by option "-q" -l= Position index of data in the input file (default: 1) - --import Import data from stdin. If "-f" is also specified, data is imported from the file instead - --export Export kintone data to stdout -v, --version Version of cli-kintone Help Options: @@ -109,14 +109,15 @@ Japanese: https://developer.cybozu.io/hc/ja/articles/202957070 ## How to Build -### Requirement +Requirement - Go 1.13.3 - Git and Mercurial to be able to clone the packages -### [Mac OS X/Linux](./docs/BuildForMacLinux.md) +[Mac OS X/Linux](./docs/BuildForMacLinux.md) + +[Windows](./docs/BuildForWindows.md) -### [Windows](./docs/BuildForWindows.md) ## License GPL v2 From 90af4934afaf8f30cbce782316a0ee801915df1c Mon Sep 17 00:00:00 2001 From: minhhy2801 Date: Mon, 9 Mar 2020 16:59:25 +0700 Subject: [PATCH 16/17] merge dev_merged_draft --- .travis.yml | 2 +- README.md | 3 +-- import-with-bulkRequest.go | 6 ++++-- main.go | 35 ++++++++--------------------------- utils.go | 20 ++++++++++++++++++++ 5 files changed, 34 insertions(+), 32 deletions(-) create mode 100644 utils.go diff --git a/.travis.yml b/.travis.yml index ec3064f..937f1e5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,6 @@ language: go go: - - 1.13.3 + - 1.13.7 env: "PATH=/home/travis/gopath/bin:$PATH" before_install: - go get github.com/mattn/gom diff --git a/README.md b/README.md index a589d15..cda118c 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,6 @@ cli-kintone is a command line utility for exporting and importing kintone App da 0.10.2 - ## Downloads These binaries are available for download. @@ -35,7 +34,7 @@ https://github.com/kintone/cli-kintone/releases -o= Output format. Specify either 'json' or 'csv' (default: csv) -e= Character encoding (default: utf-8). Only support the encoding below both field code and data itself: - 'utf-8', 'utf-16', 'utf-16be-with-signature', 'utf-16le-with-signature', 'sjis' or'euc-jp' + 'utf-8', 'utf-16', 'utf-16be-with-signature', 'utf-16le-with-signature', 'sjis' or'euc-jp', 'gbk' or 'big5' -U= Basic authentication user name -P= Basic authentication password -q= Query string diff --git a/import-with-bulkRequest.go b/import-with-bulkRequest.go index e72d7a2..67782cb 100644 --- a/import-with-bulkRequest.go +++ b/import-with-bulkRequest.go @@ -23,11 +23,13 @@ type SubRecord struct { } func getReader(reader io.Reader) io.Reader { + readerWithoutBOM := removeBOMCharacter(reader) + encoding := getEncoding() if encoding == nil { - return reader + return readerWithoutBOM } - return transform.NewReader(reader, encoding.NewDecoder()) + return transform.NewReader(readerWithoutBOM, encoding.NewDecoder()) } // delete specific records diff --git a/main.go b/main.go index 972d264..1d5b32c 100644 --- a/main.go +++ b/main.go @@ -10,8 +10,10 @@ import ( "github.com/howeyc/gopass" "github.com/kintone/go-kintone" "golang.org/x/text/encoding" - "golang.org/x/text/encoding/japanese" "golang.org/x/text/encoding/unicode" + "golang.org/x/text/encoding/japanese" + "golang.org/x/text/encoding/simplifiedchinese" + "golang.org/x/text/encoding/traditionalchinese" flags "github.com/jessevdk/go-flags" ) @@ -39,7 +41,7 @@ type Configure struct { APIToken string `short:"t" default:"" description:"API token"` GuestSpaceID uint64 `short:"g" default:"0" description:"Guest Space ID"` Format string `short:"o" default:"csv" description:"Output format. Specify either 'json' or 'csv'"` - Encoding string `short:"e" default:"utf-8" description:"Character encoding (default: utf-8).\n Only support the encoding below both field code and data itself: \n 'utf-8', 'utf-16', 'utf-16be-with-signature', 'utf-16le-with-signature', 'sjis' or 'euc-jp'"` + Encoding string `short:"e" default:"utf-8" description:"Character encoding (default: utf-8).\n Only support the encoding below both field code and data itself: \n 'utf-8', 'utf-16', 'utf-16be-with-signature', 'utf-16le-with-signature', 'sjis' or 'euc-jp', 'gbk' or 'big5'"` BasicAuthUser string `short:"U" default:"" description:"Basic authentication user name"` BasicAuthPassword string `short:"P" default:"" description:"Basic authentication password"` Query string `short:"q" default:"" description:"Query string"` @@ -77,31 +79,6 @@ type Cell struct { // Row config type Row []*Cell -func (p Columns) Len() int { - return len(p) -} - -func (p Columns) Swap(i, j int) { - p[i], p[j] = p[j], p[i] -} - -func (p Columns) Less(i, j int) bool { - p1 := p[i] - code1 := p1.Code - if p1.IsSubField { - code1 = p1.Table - } - p2 := p[j] - code2 := p2.Code - if p2.IsSubField { - code2 = p2.Table - } - if code1 == code2 { - return p[i].Code < p[j].Code - } - return code1 < code2 -} - func getFields(app *kintone.App) (map[string]*kintone.FieldInfo, error) { fields, err := app.Fields() if err != nil { @@ -196,6 +173,10 @@ func getEncoding() encoding.Encoding { return japanese.EUCJP case "sjis": return japanese.ShiftJIS + case "gbk": + return simplifiedchinese.GBK + case "big5": + return traditionalchinese.Big5 default: return nil } diff --git a/utils.go b/utils.go new file mode 100644 index 0000000..bdfa3b8 --- /dev/null +++ b/utils.go @@ -0,0 +1,20 @@ +package main + +import ( + "bufio" + "log" + "io" +) + +func removeBOMCharacter(reader io.Reader) io.Reader { + bufferReader := bufio.NewReader(reader) + r, _, err := bufferReader.ReadRune() + if err != nil { + log.Fatal(err) + } + + if r != '\uFEFF' { + bufferReader.UnreadRune() // Not a BOM -- put the rune back + } + return bufferReader +} From c033c3362e39b694f50603a97ff560f381ade887 Mon Sep 17 00:00:00 2001 From: minhhy2801 Date: Thu, 12 Mar 2020 10:22:48 +0700 Subject: [PATCH 17/17] update version 0.11.0 --- README.md | 2 +- main.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 4459212..b00fa14 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ cli-kintone is a command line utility for exporting and importing kintone App da ## Version -0.10.2 +0.11.0 ## Downloads diff --git a/main.go b/main.go index 1d5b32c..a240db6 100644 --- a/main.go +++ b/main.go @@ -22,7 +22,7 @@ import ( const NAME = "cli-kintone" // VERSION of this package -const VERSION = "0.10.2" +const VERSION = "0.11.0" // IMPORT_ROW_LIMIT The maximum row will be import const IMPORT_ROW_LIMIT = 100