Skip to content

Commit 0b7f12f

Browse files
Merge pull request #380 from abyssparanoia/feature/add-mapping-table-for-uft-to-shift-jis
Feature/add mapping table for uft to shift jis
2 parents 254991c + d9c9395 commit 0b7f12f

File tree

3 files changed

+55
-4
lines changed

3 files changed

+55
-4
lines changed

go.mod

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ require (
1212
github.com/imdario/mergo v0.3.16
1313
github.com/pkg/sftp v1.13.6
1414
github.com/stretchr/testify v1.9.0
15+
github.com/tomtwinkle/garbledreplacer v0.2.0
1516
golang.org/x/crypto v0.23.0
1617
golang.org/x/text v0.15.0
1718
gopkg.in/go-playground/assert.v1 v1.2.1

go.sum

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,8 +15,6 @@ github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/o
1515
github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY=
1616
github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY=
1717
github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY=
18-
github.com/go-playground/validator/v10 v10.20.0 h1:K9ISHbSaI0lyB2eWMPJo+kOS/FBExVwjEviJTixqxL8=
19-
github.com/go-playground/validator/v10 v10.20.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM=
2018
github.com/go-playground/validator/v10 v10.21.0 h1:4fZA11ovvtkdgaeev9RGWPgc1uj3H8W+rNYyH/ySBb0=
2119
github.com/go-playground/validator/v10 v10.21.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM=
2220
github.com/gocarina/gocsv v0.0.0-20240520201108-78e41c74b4b1 h1:FWNFq4fM1wPfcK40yHE5UO3RUdSNPaBC+j3PokzA6OQ=
@@ -39,6 +37,8 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
3937
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
4038
github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
4139
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
40+
github.com/tomtwinkle/garbledreplacer v0.2.0 h1:Oljr85vcevpk4Juiq9a1fGHLbSwBF2SQvoxDvXdDdOQ=
41+
github.com/tomtwinkle/garbledreplacer v0.2.0/go.mod h1:j+vxbXgzLTpw0VMcV5fbsUFnmWmaCAZASEa8BmsTWOM=
4242
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
4343
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
4444
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=

internal/pkg/shiftjis_transformer/transformer.go

Lines changed: 52 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,24 +1,74 @@
11
package shiftjis_transformer
22

33
import (
4+
"bytes"
45
"reflect"
6+
"strings"
57

8+
"github.com/tomtwinkle/garbledreplacer"
69
"golang.org/x/text/encoding/japanese"
710
"golang.org/x/text/transform"
811
)
912

1013
var encoder = japanese.ShiftJIS.NewEncoder()
1114
var decoder = japanese.ShiftJIS.NewDecoder()
1215

16+
var charMap = map[rune]rune{
17+
'−': '-', // Convert full-width minus to half-width hyphen
18+
'〜': '~', // Convert full-width tilde to half-width tilde
19+
'-': '-', // Convert full-width dash to half-width hyphen
20+
' ': ' ', // Convert full-width space to half-width space
21+
'”': '"', // Convert full-width double quote to half-width
22+
'“': '"', // Convert full-width double quote to half-width
23+
'‘': '\'', // Convert full-width single quote to half-width
24+
'’': '\'', // Convert full-width single quote to half-width
25+
'¥': '\\', // Convert full-width yen mark to half-width backslash
26+
'(': '(', // Convert full-width left parenthesis to half-width
27+
')': ')', // Convert full-width right parenthesis to half-width
28+
'[': '[', // Convert full-width left square bracket to half-width
29+
']': ']', // Convert full-width right square bracket to half-width
30+
'{': '{', // Convert full-width left curly brace to half-width
31+
'}': '}', // Convert full-width right curly brace to half-width
32+
'<': '<', // Convert full-width less than to half-width
33+
'>': '>', // Convert full-width greater than to half-width
34+
'=': '=', // Convert full-width equals to half-width
35+
'+': '+', // Convert full-width plus to half-width
36+
';': ';', // Convert full-width semicolon to half-width
37+
':': ':', // Convert full-width colon to half-width
38+
'*': '*', // Convert full-width asterisk to half-width
39+
'&': '&', // Convert full-width ampersand to half-width
40+
'%': '%', // Convert full-width percent to half-width
41+
'#': '#', // Convert full-width hash to half-width
42+
'@': '@', // Convert full-width at sign to half-width
43+
'!': '!', // Convert full-width exclamation mark to half-width
44+
'?': '?', // Convert full-width question mark to half-width
45+
'|': '|', // Convert full-width vertical bar to half-width
46+
}
47+
48+
// replace specific characters in the string with the mapping table
49+
func replaceUnsupportedChars(input string) string {
50+
return strings.Map(func(r rune) rune {
51+
if newR, ok := charMap[r]; ok {
52+
return newR
53+
}
54+
return r
55+
}, input)
56+
}
57+
1358
func EncodeToShiftJISFromUTF8(input interface{}) error {
1459
val := reflect.ValueOf(input).Elem()
1560
for i := 0; i < val.NumField(); i++ {
1661
if val.Field(i).Kind() == reflect.String {
17-
encoded, _, err := transform.String(encoder, val.Field(i).String())
62+
original := val.Field(i).String()
63+
replaced := replaceUnsupportedChars(original)
64+
65+
var buf bytes.Buffer
66+
w := transform.NewWriter(&buf, garbledreplacer.NewTransformer(japanese.ShiftJIS, '?'))
67+
_, err := w.Write([]byte(replaced))
1868
if err != nil {
1969
return err
2070
}
21-
val.Field(i).SetString(encoded)
71+
val.Field(i).SetString(buf.String())
2272
}
2373
}
2474
return nil

0 commit comments

Comments
 (0)