From 9cbc8a8bb110704c3ed3d185cb484058359067e6 Mon Sep 17 00:00:00 2001 From: Mike Date: Thu, 6 Jun 2024 21:36:45 -0400 Subject: [PATCH] Adds optimizations to tokenizer and parser. --- main/main.go | 2 +- pkg/operations/constants/constant.go | 4 +-- pkg/operations/functions/abs.go | 4 +-- pkg/operations/functions/addition.go | 4 +-- pkg/operations/functions/arccos.go | 4 +-- pkg/operations/functions/arcsin.go | 4 +-- pkg/operations/functions/arctan.go | 4 +-- pkg/operations/functions/cos.go | 4 +-- pkg/operations/functions/division.go | 4 +-- pkg/operations/functions/log.go | 6 ++-- pkg/operations/functions/multiplication.go | 4 +-- pkg/operations/functions/negation.go | 4 +-- pkg/operations/functions/parentheses.go | 4 +-- pkg/operations/functions/power.go | 4 +-- pkg/operations/functions/sin.go | 4 +-- pkg/operations/functions/subtraction.go | 4 +-- pkg/operations/functions/tan.go | 4 +-- pkg/operations/parser/parser.go | 18 +++++------ pkg/operations/parser/parser_test.go | 13 ++++++-- pkg/operations/parser/stress_test.go | 31 +++++++++++++++++++ pkg/operations/parser/tokens/tokenizer.go | 8 ++--- .../parser/tokens/tokenizer_test.go | 6 ++++ pkg/utils/utils.go | 20 ++++++++++++ 23 files changed, 115 insertions(+), 49 deletions(-) create mode 100644 pkg/operations/parser/stress_test.go diff --git a/main/main.go b/main/main.go index f45c567..46ef5f3 100644 --- a/main/main.go +++ b/main/main.go @@ -14,7 +14,7 @@ func main() { func TestParse() { var inputString = "a * b * c + 123 + sin(x)" - var parsed, err = parser.Parse(inputString) + var parsed, err = parser.ParseOperation(inputString) if err != nil { fmt.Println((*err).Error()) return diff --git a/pkg/operations/constants/constant.go b/pkg/operations/constants/constant.go index 5c11b41..d4b8df1 100644 --- a/pkg/operations/constants/constant.go +++ b/pkg/operations/constants/constant.go @@ -2,7 +2,7 @@ package constants import ( "goast/pkg/operations" - "strconv" + "goast/pkg/utils" ) type Constant struct { @@ -22,7 +22,7 @@ func (c Constant) ToString() string { if c.StringRepresentation != nil { return *(c.StringRepresentation) } - return strconv.FormatComplex(c.Representation, 'G', 5, 64) + return utils.SmartComplexString(c.Representation) } func (c Constant) ToNumber() complex128 { diff --git a/pkg/operations/functions/abs.go b/pkg/operations/functions/abs.go index 5d5af6f..ede9cbf 100644 --- a/pkg/operations/functions/abs.go +++ b/pkg/operations/functions/abs.go @@ -2,8 +2,8 @@ package functions import ( "goast/pkg/operations" + "goast/pkg/utils" "math/cmplx" - "strconv" ) type Abs struct { @@ -21,7 +21,7 @@ func (a Abs) IsConstant() bool { func (a Abs) ToString() string { if a.Inner.IsConstant() { c := a.ToNumber() - return strconv.FormatComplex(c, 'f', 5, 64) + return utils.SmartComplexString(c) } retString := "abs(" retString += a.Inner.ToString() diff --git a/pkg/operations/functions/addition.go b/pkg/operations/functions/addition.go index 3d81742..30d225f 100644 --- a/pkg/operations/functions/addition.go +++ b/pkg/operations/functions/addition.go @@ -2,8 +2,8 @@ package functions import ( "goast/pkg/operations" + "goast/pkg/utils" "math/cmplx" - "strconv" ) type Addition struct { @@ -27,7 +27,7 @@ func (a Addition) IsConstant() bool { func (a Addition) ToString() string { if a.IsConstant() { c := a.ToNumber() - return strconv.FormatComplex(c, 'f', 5, 64) + return utils.SmartComplexString(c) } retString := "" var values = a.GetValues() diff --git a/pkg/operations/functions/arccos.go b/pkg/operations/functions/arccos.go index 578b237..49b1bc1 100644 --- a/pkg/operations/functions/arccos.go +++ b/pkg/operations/functions/arccos.go @@ -2,8 +2,8 @@ package functions import ( "goast/pkg/operations" + "goast/pkg/utils" "math/cmplx" - "strconv" ) type ArcCos struct { @@ -21,7 +21,7 @@ func (a ArcCos) IsConstant() bool { func (a ArcCos) ToString() string { if a.Inner.IsConstant() { c := a.ToNumber() - return strconv.FormatComplex(c, 'f', 5, 64) + return utils.SmartComplexString(c) } retString := "arccos(" retString += a.Inner.ToString() diff --git a/pkg/operations/functions/arcsin.go b/pkg/operations/functions/arcsin.go index 3df2850..ad33c5d 100644 --- a/pkg/operations/functions/arcsin.go +++ b/pkg/operations/functions/arcsin.go @@ -2,8 +2,8 @@ package functions import ( "goast/pkg/operations" + "goast/pkg/utils" "math/cmplx" - "strconv" ) type ArcSin struct { @@ -21,7 +21,7 @@ func (a ArcSin) IsConstant() bool { func (a ArcSin) ToString() string { if a.Inner.IsConstant() { c := a.ToNumber() - return strconv.FormatComplex(c, 'f', 5, 64) + return utils.SmartComplexString(c) } retString := "arcsin(" retString += a.Inner.ToString() diff --git a/pkg/operations/functions/arctan.go b/pkg/operations/functions/arctan.go index 7a3fc2a..dec57a1 100644 --- a/pkg/operations/functions/arctan.go +++ b/pkg/operations/functions/arctan.go @@ -2,8 +2,8 @@ package functions import ( "goast/pkg/operations" + "goast/pkg/utils" "math/cmplx" - "strconv" ) type ArcTan struct { @@ -21,7 +21,7 @@ func (a ArcTan) IsConstant() bool { func (a ArcTan) ToString() string { if a.Inner.IsConstant() { c := a.ToNumber() - return strconv.FormatComplex(c, 'f', 5, 64) + return utils.SmartComplexString(c) } retString := "arctan(" retString += a.Inner.ToString() diff --git a/pkg/operations/functions/cos.go b/pkg/operations/functions/cos.go index f4c70ce..992c00f 100644 --- a/pkg/operations/functions/cos.go +++ b/pkg/operations/functions/cos.go @@ -2,8 +2,8 @@ package functions import ( "goast/pkg/operations" + "goast/pkg/utils" "math/cmplx" - "strconv" ) type Cos struct { @@ -21,7 +21,7 @@ func (c Cos) IsConstant() bool { func (c Cos) ToString() string { if c.Inner.IsConstant() { c := c.ToNumber() - return strconv.FormatComplex(c, 'f', 5, 64) + return utils.SmartComplexString(c) } retString := "cos(" retString += c.Inner.ToString() diff --git a/pkg/operations/functions/division.go b/pkg/operations/functions/division.go index 74c7691..a78a9b0 100644 --- a/pkg/operations/functions/division.go +++ b/pkg/operations/functions/division.go @@ -2,8 +2,8 @@ package functions import ( "goast/pkg/operations" + "goast/pkg/utils" "math/cmplx" - "strconv" ) type Division struct { @@ -28,7 +28,7 @@ func (d Division) IsConstant() bool { func (d Division) ToString() string { if d.IsConstant() { c := d.ToNumber() - return strconv.FormatComplex(c, 'f', 5, 64) + return utils.SmartComplexString(c) } retString := "" retString += d.Numerator.ToString() diff --git a/pkg/operations/functions/log.go b/pkg/operations/functions/log.go index fffc04e..c3ddd39 100644 --- a/pkg/operations/functions/log.go +++ b/pkg/operations/functions/log.go @@ -2,8 +2,8 @@ package functions import ( "goast/pkg/operations" + "goast/pkg/utils" "math/cmplx" - "strconv" ) type Log struct { @@ -22,9 +22,9 @@ func (l Log) IsConstant() bool { func (l Log) ToString() string { if l.Inner.IsConstant() { c := l.ToNumber() - return strconv.FormatComplex(c, 'f', 5, 64) + return utils.SmartComplexString(c) } - retString := "log(" + retString := "log_(" retString += l.Base.ToString() + ", " retString += l.Inner.ToString() retString += ")" diff --git a/pkg/operations/functions/multiplication.go b/pkg/operations/functions/multiplication.go index 645d6aa..8f25e6f 100644 --- a/pkg/operations/functions/multiplication.go +++ b/pkg/operations/functions/multiplication.go @@ -2,8 +2,8 @@ package functions import ( "goast/pkg/operations" + "goast/pkg/utils" "math/cmplx" - "strconv" ) type Multiplication struct { @@ -27,7 +27,7 @@ func (m Multiplication) IsConstant() bool { func (m Multiplication) ToString() string { if m.IsConstant() { c := m.ToNumber() - return strconv.FormatComplex(c, 'f', 5, 64) + return utils.SmartComplexString(c) } retString := "" var values = m.GetValues() diff --git a/pkg/operations/functions/negation.go b/pkg/operations/functions/negation.go index 012c33f..c760f50 100644 --- a/pkg/operations/functions/negation.go +++ b/pkg/operations/functions/negation.go @@ -2,8 +2,8 @@ package functions import ( "goast/pkg/operations" + "goast/pkg/utils" "math/cmplx" - "strconv" ) type Negation struct { @@ -21,7 +21,7 @@ func (n Negation) IsConstant() bool { func (n Negation) ToString() string { if n.Inner.IsConstant() { c := n.ToNumber() - return strconv.FormatComplex(c, 'f', 5, 64) + return utils.SmartComplexString(c) } retString := "-" retString += n.Inner.ToString() diff --git a/pkg/operations/functions/parentheses.go b/pkg/operations/functions/parentheses.go index ebb8553..ca920fc 100644 --- a/pkg/operations/functions/parentheses.go +++ b/pkg/operations/functions/parentheses.go @@ -2,8 +2,8 @@ package functions import ( "goast/pkg/operations" + "goast/pkg/utils" "math/cmplx" - "strconv" ) type Parentheses struct { @@ -21,7 +21,7 @@ func (p Parentheses) IsConstant() bool { func (p Parentheses) ToString() string { if p.Inner.IsConstant() { c := p.Inner.ToNumber() - return strconv.FormatComplex(c, 'f', 5, 64) + return utils.SmartComplexString(c) } retString := "(" retString += p.Inner.ToString() diff --git a/pkg/operations/functions/power.go b/pkg/operations/functions/power.go index 418b531..0f1d339 100644 --- a/pkg/operations/functions/power.go +++ b/pkg/operations/functions/power.go @@ -2,8 +2,8 @@ package functions import ( "goast/pkg/operations" + "goast/pkg/utils" "math/cmplx" - "strconv" ) type Power struct { @@ -25,7 +25,7 @@ func (p Power) IsConstant() bool { func (p Power) ToString() string { if p.IsConstant() { c := p.ToNumber() - return strconv.FormatComplex(c, 'f', 5, 64) + return utils.SmartComplexString(c) } retString := p.Base.ToString() retString += " ^ " diff --git a/pkg/operations/functions/sin.go b/pkg/operations/functions/sin.go index c91a6cd..b202329 100644 --- a/pkg/operations/functions/sin.go +++ b/pkg/operations/functions/sin.go @@ -2,8 +2,8 @@ package functions import ( "goast/pkg/operations" + "goast/pkg/utils" "math/cmplx" - "strconv" ) type Sin struct { @@ -21,7 +21,7 @@ func (s Sin) IsConstant() bool { func (s Sin) ToString() string { if s.Inner.IsConstant() { c := s.ToNumber() - return strconv.FormatComplex(c, 'f', 5, 64) + return utils.SmartComplexString(c) } retString := "sin(" retString += s.Inner.ToString() diff --git a/pkg/operations/functions/subtraction.go b/pkg/operations/functions/subtraction.go index 8a1ad1d..f6355cd 100644 --- a/pkg/operations/functions/subtraction.go +++ b/pkg/operations/functions/subtraction.go @@ -2,8 +2,8 @@ package functions import ( "goast/pkg/operations" + "goast/pkg/utils" "math/cmplx" - "strconv" ) type Subtraction struct { @@ -27,7 +27,7 @@ func (s Subtraction) IsConstant() bool { func (s Subtraction) ToString() string { if s.IsConstant() { c := s.ToNumber() - return strconv.FormatComplex(c, 'f', 5, 64) + return utils.SmartComplexString(c) } retString := "" for i := 0; i < len(s.Values); i++ { diff --git a/pkg/operations/functions/tan.go b/pkg/operations/functions/tan.go index 2beeb41..9ff4203 100644 --- a/pkg/operations/functions/tan.go +++ b/pkg/operations/functions/tan.go @@ -2,8 +2,8 @@ package functions import ( "goast/pkg/operations" + "goast/pkg/utils" "math/cmplx" - "strconv" ) type Tan struct { @@ -21,7 +21,7 @@ func (t Tan) IsConstant() bool { func (t Tan) ToString() string { if t.Inner.IsConstant() { c := t.ToNumber() - return strconv.FormatComplex(c, 'f', 5, 64) + return utils.SmartComplexString(c) } retString := "tan(" retString += t.Inner.ToString() diff --git a/pkg/operations/parser/parser.go b/pkg/operations/parser/parser.go index feffad7..9fd936c 100644 --- a/pkg/operations/parser/parser.go +++ b/pkg/operations/parser/parser.go @@ -423,7 +423,7 @@ func Remove(intermediates []Intermediate, other Intermediate) []Intermediate { return intermediates } -func Parse(inputString string) (operations.Operation, *error) { +func ParseOperation(inputString string) (operations.Operation, *error) { err := validateString(inputString) if err != nil { return constants.NaN, err @@ -730,7 +730,7 @@ func generateOperators(current []Intermediate, inputList []tokens.Token) []Inter } func generateIdentities(current []Intermediate, inputList []tokens.Token) []Intermediate { - var clone = slices.Clone(current) + var clone = current for i, token := range inputList { if indexProcessedOperation(i, clone) == false { if token.TokenType == tokens.Operator { @@ -762,13 +762,13 @@ func generateIdentities(current []Intermediate, inputList []tokens.Token) []Inte } } } - var retList = slices.Clone(clone) + var retList = clone SortByStartIndex(retList) return retList } func generatePowers(current []Intermediate, inputList []tokens.Token) []Intermediate { - var clone = slices.Clone(current) + var clone = current for i, token := range inputList { if indexProcessedOperation(i, clone) == false { if token.TokenType == tokens.Operator { @@ -790,14 +790,14 @@ func generatePowers(current []Intermediate, inputList []tokens.Token) []Intermed } } } - var retList = slices.Clone(clone) + var retList = clone SortByStartIndex(retList) return retList } func generateMultiplicationAndDivision(current []Intermediate, inputList []tokens.Token) []Intermediate { // TODO Require left and right checks for mult and div - var clone = slices.Clone(current) + var clone = current for i, token := range inputList { if indexProcessedOperation(i, clone) == false { if token.TokenType == tokens.Operator { @@ -831,14 +831,14 @@ func generateMultiplicationAndDivision(current []Intermediate, inputList []token } } } - var retList = slices.Clone(clone) + var retList = clone SortByStartIndex(retList) return retList } func generateAdditionAndSubtraction(current []Intermediate, inputList []tokens.Token) []Intermediate { // TODO Require left and right checks for add and sub - var clone = slices.Clone(current) + var clone = current for i, token := range inputList { if indexProcessedOperation(i, clone) == false { if token.TokenType == tokens.Operator { @@ -873,7 +873,7 @@ func generateAdditionAndSubtraction(current []Intermediate, inputList []tokens.T } } - var retList = slices.Clone(clone) + var retList = clone SortByStartIndex(retList) return retList } diff --git a/pkg/operations/parser/parser_test.go b/pkg/operations/parser/parser_test.go index a7e6bfd..867f834 100644 --- a/pkg/operations/parser/parser_test.go +++ b/pkg/operations/parser/parser_test.go @@ -8,9 +8,9 @@ import ( "testing" ) -func TestParse(t *testing.T) { +func TestParse1(t *testing.T) { var inputString = "a + 123 + sin(x)" - var parsed, err = Parse(inputString) + var parsed, err = ParseOperation(inputString) if err != nil { t.Error(*err) } @@ -20,3 +20,12 @@ func TestParse(t *testing.T) { parsed = parsed.Evaluate(variables.Variable{Name: "a"}, constants.TEN) fmt.Println(parsed.ToString()) } + +func TestParse2(t *testing.T) { + var inputString = "log_(x,y)" + var parsed, err = ParseOperation(inputString) + if err != nil { + t.Error(*err) + } + fmt.Println(parsed.ToString()) +} diff --git a/pkg/operations/parser/stress_test.go b/pkg/operations/parser/stress_test.go new file mode 100644 index 0000000..cb0ad43 --- /dev/null +++ b/pkg/operations/parser/stress_test.go @@ -0,0 +1,31 @@ +package parser + +import ( + "fmt" + "goast/pkg/utils" + "strconv" + "testing" + "time" +) + +func TestStress1(t *testing.T) { + numIterations := 20 + times := make([]int64, numIterations) + stringSize := 500 + inputString := "" + for i := 0; i < stringSize; i++ { + inputString += strconv.Itoa(i) + if i < stringSize-1 { + inputString += " + " + } + } + fmt.Println(inputString) + for i := 0; i < numIterations; i++ { + start := time.Now().UnixMilli() + ParseOperation(inputString) + end := time.Now().UnixMilli() + times[i] = end - start + } + stringRep := utils.SliceToString(times) + fmt.Println(stringRep) +} diff --git a/pkg/operations/parser/tokens/tokenizer.go b/pkg/operations/parser/tokens/tokenizer.go index 7f3d047..b1debfa 100644 --- a/pkg/operations/parser/tokens/tokenizer.go +++ b/pkg/operations/parser/tokens/tokenizer.go @@ -12,7 +12,7 @@ import ( const ( Decimal = '.' ValidNumbers = "0123456789." - Operators = "+-*/^" + Operators = "+-*/^," OpenParenthesisCharacter = '(' ClosedParenthesisCharacter = ')' ) @@ -74,7 +74,7 @@ func tokenizeNumbers(inputString string) []Token { } func tokenizeOperators(tokenList []Token, inputString string) []Token { - retList := slices.Clone(tokenList) + retList := tokenList for i, v := range inputString { if IndexProcessed(i, retList) == false { if strings.ContainsRune(Operators, v) { @@ -89,7 +89,7 @@ func tokenizeOperators(tokenList []Token, inputString string) []Token { } func tokenizeParentheses(tokenList []Token, inputString string) []Token { - retList := slices.Clone(tokenList) + retList := tokenList for i, v := range inputString { if IndexProcessed(i, retList) == false { if v == OpenParenthesisCharacter { @@ -109,7 +109,7 @@ func tokenizeParentheses(tokenList []Token, inputString string) []Token { } func tokenizeText(tokenList []Token, inputString string) []Token { - retList := slices.Clone(tokenList) + retList := tokenList var accumulated = "" for i, v := range inputString { if IndexProcessed(i, retList) { diff --git a/pkg/operations/parser/tokens/tokenizer_test.go b/pkg/operations/parser/tokens/tokenizer_test.go index 163761d..02f08c4 100644 --- a/pkg/operations/parser/tokens/tokenizer_test.go +++ b/pkg/operations/parser/tokens/tokenizer_test.go @@ -27,3 +27,9 @@ func TestTokenize4(t *testing.T) { var result = Tokenize(inputString) fmt.Println(ToString(result)) } + +func TestTokenize5(t *testing.T) { + var inputString = "log_(x,y)" + var result = Tokenize(inputString) + fmt.Println(ToString(result)) +} diff --git a/pkg/utils/utils.go b/pkg/utils/utils.go index 53c654f..1f4414f 100644 --- a/pkg/utils/utils.go +++ b/pkg/utils/utils.go @@ -2,6 +2,7 @@ package utils import ( "reflect" + "strconv" "strings" "unicode/utf8" ) @@ -135,3 +136,22 @@ func Keys[M ~map[K]V, K comparable, V any](m M) []K { } return r } + +func SmartComplexString(c complex128) string { + if imag(c) == 0 { + return strconv.FormatFloat(real(c), 'G', 5, 64) + } + return strconv.FormatComplex(c, 'G', 5, 64) +} + +func SliceToString(array []int64) string { + retString := "[" + for i, v := range array { + retString += strconv.Itoa(int(v)) + if i < len(array)-1 { + retString += ", " + } + } + retString += "]" + return retString +}