diff --git a/Makefile b/Makefile index ba5749c..dc78dde 100644 --- a/Makefile +++ b/Makefile @@ -21,6 +21,8 @@ tidy: go.sum test_all: $(GINKGO) run -r ./ +validate_codecov: .make/validate_codecov + cover: cover.profile go tool cover -func=$< @@ -57,3 +59,7 @@ bin/devctl: .versions/devctl .make/test: $(shell $(DEVCTL) list --go) | bin/ginkgo bin/devctl $(GINKGO) run ${TEST_FLAGS} $(sort $(dir $?)) @touch $@ + +.make/validate_codecov: codecov.yml + curl -X POST --data-binary @codecov.yml https://codecov.io/validate + @touch $@ diff --git a/README.md b/README.md index 2f34886..e3e0fa6 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,20 @@ Makefile parsing and utilities in Go ## Usage -At present the scanning utilities are the most tested. +### Reading + +The `make.Parser` is the primary way to read Makefiles. + +```go +f := os.Open("Makefile") +p := make.NewParser(f, nil) + +m, err := p.ParseFile() + +fmt.Println(m.Rules) +``` + +The more primitive `make.Scanner` and `make.ScanTokens` used by `make.Parser` can be used individually. Using `make.ScanTokens` with a `bufio.Scanner` @@ -23,19 +36,15 @@ Using `make.Scanner` ```go f := os.Open("Makefile") -s := make.NewScanner(f) +s := make.NewScanner(f, nil) -for s.Scan() { - s.Token() // The current token.Token i.e. token.SIMPLE_ASSIGN - s.Literal() // Literal tokens as a string i.e. "identifier" +for pos, tok, lit := s.Scan(); tok != token.EOF { + fmt.Println(pos) // The position of tok + fmt.Println(tok) // The current token.Token i.e. token.SIMPLE_ASSIGN + fmt.Println(lit) // Literal tokens as a string i.e. "identifier" } if err := s.Err(); err != nil { fmt.Println(err) } ``` - -## Future - -- `make.Parser` -- `make.Parse(file)` diff --git a/ast/ast.go b/ast/ast.go index ceaffad..47c1118 100644 --- a/ast/ast.go +++ b/ast/ast.go @@ -8,8 +8,6 @@ import ( type Node = ast.Node -var Walk = ast.Walk - // A File represents text content interpreted as the make syntax. // Most commonly this is a Makefile, but could also be any file // understood by make, i.e. include-me.mk @@ -79,6 +77,11 @@ type TargetList struct { List []FileName } +// Add appends target to t.List +func (t *TargetList) Add(target FileName) { + t.List = append(t.List, target) +} + // Pos implements Node func (t *TargetList) Pos() token.Pos { return t.List[0].Pos() @@ -95,6 +98,11 @@ type PreReqList struct { List []FileName } +// Add appends prereq to p.List +func (p *PreReqList) Add(prereq FileName) { + p.List = append(p.List, prereq) +} + // Pos implements Node func (p *PreReqList) Pos() token.Pos { return p.List[0].Pos() diff --git a/ast/ast_test.go b/ast/ast_test.go index 6809026..44a85c5 100644 --- a/ast/ast_test.go +++ b/ast/ast_test.go @@ -60,10 +60,10 @@ var _ = Describe("Ast", func() { It("should return the position after the final recipe", func() { c := &ast.Rule{Recipes: []*ast.Recipe{{ TokPos: token.Pos(420), + Text: "some text", }}} - // TODO: This is wrong, should be position after text - Expect(c.End()).To(Equal(token.Pos(420))) + Expect(c.End()).To(Equal(token.Pos(429))) }) }) @@ -89,6 +89,17 @@ var _ = Describe("Ast", func() { Expect(c.End()).To(Equal(token.Pos(423))) }) + + It("should append the given target", func() { + c := &ast.TargetList{} + elem := &ast.LiteralFileName{Name: &ast.Ident{ + NamePos: token.Pos(69), + }} + + c.Add(elem) + + Expect(c.List).To(ContainElement(elem)) + }) }) Describe("PreReqList", func() { @@ -102,7 +113,7 @@ var _ = Describe("Ast", func() { Expect(c.Pos()).To(Equal(token.Pos(69))) }) - It("should return the position after the lat prereq", func() { + It("should return the position after the last prereq", func() { c := &ast.PreReqList{List: []ast.FileName{ &ast.LiteralFileName{Name: &ast.Ident{NamePos: token.Pos(69)}}, &ast.LiteralFileName{Name: &ast.Ident{ @@ -113,6 +124,17 @@ var _ = Describe("Ast", func() { Expect(c.End()).To(Equal(token.Pos(423))) }) + + It("should append the given prereq", func() { + c := &ast.PreReqList{} + elem := &ast.LiteralFileName{Name: &ast.Ident{ + NamePos: token.Pos(69), + }} + + c.Add(elem) + + Expect(c.List).To(ContainElement(elem)) + }) }) Describe("LiteralFileName", func() { diff --git a/codecov.yml b/codecov.yml new file mode 100644 index 0000000..c5d7535 --- /dev/null +++ b/codecov.yml @@ -0,0 +1,9 @@ +coverage: + status: + project: + default: + target: 75% + threshold: 3% + patch: + target: 60% + threshold: 10% diff --git a/parser.go b/parser.go index eb94e9d..419cf0b 100644 --- a/parser.go +++ b/parser.go @@ -3,6 +3,8 @@ package make import ( "go/scanner" "io" + "math" + "strings" "github.com/unmango/go-make/ast" "github.com/unmango/go-make/token" @@ -16,12 +18,20 @@ type Parser struct { pos token.Pos tok token.Token // one token look-ahead lit string // token literal + + recipePrefix token.Token } func NewParser(r io.Reader, file *token.File) *Parser { + if file == nil { + file = token.NewFileSet().AddFile("", 1, math.MaxInt-2) + } + p := &Parser{ s: NewScanner(r, file), file: file, + + recipePrefix: token.TAB, } p.next() @@ -38,21 +48,35 @@ func (p *Parser) ParseFile() (*ast.File, error) { } } +func (p *Parser) error(pos token.Pos, msg string) { + epos := p.file.Position(pos) + p.errors.Add(epos, msg) +} + +func (p *Parser) errorExpected(pos token.Pos, msg string) { + msg = "expected " + msg + if p.pos == pos { + switch { + case p.tok.IsLiteral(): + msg += ", found " + p.lit + default: + msg += ", found '" + p.tok.String() + "'" + } + } + + p.error(pos, msg) +} + func (p *Parser) expect(tok token.Token) token.Pos { pos := p.pos if p.tok != tok { - p.error(pos, "expected '"+tok.String()+"'") + p.errorExpected(pos, "'"+tok.String()+"'") } p.next() return pos } -func (p *Parser) error(pos token.Pos, msg string) { - epos := p.file.Position(pos) - p.errors.Add(epos, msg) -} - func (p *Parser) next() { p.pos, p.tok, p.lit = p.s.Scan() } @@ -76,14 +100,12 @@ func (p *Parser) parseFile() *ast.File { } func (p *Parser) parseRule() *ast.Rule { - if p.tok != token.IDENT { - p.expect(token.IDENT) - return nil - } - - var targets []ast.FileName + targets := new(ast.TargetList) for p.tok != token.COLON && p.tok != token.EOF { - targets = append(targets, p.parseFileName()) + targets.Add(p.parseFileName()) + } + if p.errors.Len() > 0 { + return nil } var colon token.Pos @@ -93,22 +115,72 @@ func (p *Parser) parseRule() *ast.Rule { } else { p.expect(token.COLON) } + if p.errors.Len() > 0 { + return nil + } + + prereqs := new(ast.PreReqList) + for p.tok != token.NEWLINE && p.tok != token.EOF { + prereqs.Add(p.parseFileName()) + } + if p.errors.Len() > 0 { + return nil + } + if p.tok == token.NEWLINE { + p.next() + } + + recipes := make([]*ast.Recipe, 0) + for p.isRecipePrefix() && p.tok != token.EOF { + recipes = append(recipes, p.parseRecipe()) + } + if p.errors.Len() > 0 { + return nil + } return &ast.Rule{ - Targets: &ast.TargetList{ - List: targets, - }, + Targets: targets, Colon: colon, Pipe: token.NoPos, Semi: token.NoPos, - PreReqs: &ast.PreReqList{}, - Recipes: []*ast.Recipe{}, + PreReqs: prereqs, + Recipes: recipes, + } +} + +func (p Parser) isRecipePrefix() bool { + return p.tok == p.recipePrefix +} + +func (p *Parser) parseRecipe() *ast.Recipe { + if !p.isRecipePrefix() { + p.expect(p.recipePrefix) + return nil + } + + tokPos := p.pos + b := &strings.Builder{} + p.next() + for p.tok != token.NEWLINE && p.tok != token.EOF { + b.WriteString(p.lit) + p.next() + } + if p.tok == token.NEWLINE { + p.next() + } + + return &ast.Recipe{ + Tok: token.TAB, + TokPos: tokPos, + Text: b.String(), } } func (p *Parser) parseFileName() ast.FileName { + name := p.parseIdent() + return &ast.LiteralFileName{ - Name: p.parseIdent(), + Name: name, } } diff --git a/parser_test.go b/parser_test.go index 64a4fea..b7561fa 100644 --- a/parser_test.go +++ b/parser_test.go @@ -2,7 +2,6 @@ package make_test import ( "bytes" - "go/token" gotoken "go/token" "math" @@ -10,6 +9,8 @@ import ( . "github.com/onsi/gomega" "github.com/unmango/go-make" + "github.com/unmango/go-make/ast" + "github.com/unmango/go-make/token" ) var _ = Describe("Parser", func() { @@ -26,17 +27,205 @@ var _ = Describe("Parser", func() { f, err := p.ParseFile() Expect(err).NotTo(HaveOccurred()) - Expect(f).NotTo(BeNil()) + Expect(f.Rules).To(ConsistOf(&ast.Rule{ + Colon: token.Pos(7), + Targets: &ast.TargetList{List: []ast.FileName{ + &ast.LiteralFileName{Name: &ast.Ident{ + Name: "target", + NamePos: token.Pos(1), + }}, + }}, + PreReqs: &ast.PreReqList{}, + Recipes: []*ast.Recipe{}, + })) }) - It("should error when starting at a colon", func() { - buf := bytes.NewBufferString(":") + It("should Parse a rule with multiple targets", func() { + buf := bytes.NewBufferString("target target2:") p := make.NewParser(buf, file) - _, err := p.ParseFile() + f, err := p.ParseFile() + + Expect(err).NotTo(HaveOccurred()) + Expect(f.Rules).To(ConsistOf(&ast.Rule{ + Colon: token.Pos(15), + Targets: &ast.TargetList{List: []ast.FileName{ + &ast.LiteralFileName{Name: &ast.Ident{ + Name: "target", + NamePos: token.Pos(1), + }}, + &ast.LiteralFileName{Name: &ast.Ident{ + Name: "target2", + NamePos: token.Pos(8), + }}, + }}, + PreReqs: &ast.PreReqList{}, + Recipes: []*ast.Recipe{}, + })) + }) + + It("should Parse a target with a prereq", func() { + buf := bytes.NewBufferString("target: prereq") + p := make.NewParser(buf, file) + + f, err := p.ParseFile() + + Expect(err).NotTo(HaveOccurred()) + Expect(f.Rules).To(ConsistOf(&ast.Rule{ + Colon: token.Pos(7), + Targets: &ast.TargetList{List: []ast.FileName{ + &ast.LiteralFileName{Name: &ast.Ident{ + Name: "target", + NamePos: token.Pos(1), + }}, + }}, + PreReqs: &ast.PreReqList{List: []ast.FileName{ + &ast.LiteralFileName{Name: &ast.Ident{ + Name: "prereq", + NamePos: token.Pos(9), + }}, + }}, + Recipes: []*ast.Recipe{}, + })) + }) + + It("should Parse a target with multiple prereqs", func() { + buf := bytes.NewBufferString("target: prereq prereq2") + p := make.NewParser(buf, file) + + f, err := p.ParseFile() + + Expect(err).NotTo(HaveOccurred()) + Expect(f.Rules).To(ConsistOf(&ast.Rule{ + Colon: token.Pos(7), + Targets: &ast.TargetList{List: []ast.FileName{ + &ast.LiteralFileName{Name: &ast.Ident{ + Name: "target", + NamePos: token.Pos(1), + }}, + }}, + PreReqs: &ast.PreReqList{List: []ast.FileName{ + &ast.LiteralFileName{Name: &ast.Ident{ + Name: "prereq", + NamePos: token.Pos(9), + }}, + &ast.LiteralFileName{Name: &ast.Ident{ + Name: "prereq2", + NamePos: token.Pos(16), + }}, + }}, + Recipes: []*ast.Recipe{}, + })) + }) + + It("should Parse a target with a recipe", func() { + buf := bytes.NewBufferString("target:\n\trecipe") + p := make.NewParser(buf, file) + + f, err := p.ParseFile() + + Expect(err).NotTo(HaveOccurred()) + Expect(f.Rules).To(ConsistOf(&ast.Rule{ + Colon: token.Pos(7), + Targets: &ast.TargetList{List: []ast.FileName{ + &ast.LiteralFileName{Name: &ast.Ident{ + Name: "target", + NamePos: token.Pos(1), + }}, + }}, + PreReqs: &ast.PreReqList{}, + Recipes: []*ast.Recipe{{ + Tok: token.TAB, + TokPos: token.Pos(9), + Text: "recipe", + }}, + })) + }) + + It("should Parse a target with multiple recipes", func() { + buf := bytes.NewBufferString("target:\n\trecipe\n\trecipe2") + p := make.NewParser(buf, file) + + f, err := p.ParseFile() - Expect(err).To(MatchError( - ContainSubstring("expected 'IDENT'"), - )) + Expect(err).NotTo(HaveOccurred()) + Expect(f.Rules).To(ConsistOf(&ast.Rule{ + Colon: token.Pos(7), + Targets: &ast.TargetList{List: []ast.FileName{ + &ast.LiteralFileName{Name: &ast.Ident{ + Name: "target", + NamePos: token.Pos(1), + }}, + }}, + PreReqs: &ast.PreReqList{}, + Recipes: []*ast.Recipe{ + { + Tok: token.TAB, + TokPos: token.Pos(9), + Text: "recipe", + }, + { + Tok: token.TAB, + TokPos: token.Pos(17), + Text: "recipe2", + }, + }, + })) + }) + + It("should Parse a target with a prereq and a recipe", func() { + buf := bytes.NewBufferString("target: prereq\n\trecipe") + p := make.NewParser(buf, file) + + f, err := p.ParseFile() + + Expect(err).NotTo(HaveOccurred()) + Expect(f.Rules).To(ConsistOf(&ast.Rule{ + Colon: token.Pos(7), + Targets: &ast.TargetList{List: []ast.FileName{ + &ast.LiteralFileName{Name: &ast.Ident{ + Name: "target", + NamePos: token.Pos(1), + }}, + }}, + PreReqs: &ast.PreReqList{List: []ast.FileName{ + &ast.LiteralFileName{Name: &ast.Ident{ + Name: "prereq", + NamePos: token.Pos(9), + }}, + }}, + Recipes: []*ast.Recipe{{ + Tok: token.TAB, + TokPos: token.Pos(16), + Text: "recipe", + }}, + })) + }) + + DescribeTable("should error on invalid starting token", + Entry(nil, ","), + Entry(nil, ";"), + Entry(nil, "|"), + Entry(nil, "="), + func(input string) { + buf := bytes.NewBufferString(input) + p := make.NewParser(buf, file) + + _, err := p.ParseFile() + + Expect(err).To(MatchError( + ContainSubstring("expected 'IDENT'"), + )) + }, + ) + + It("should support a nil *token.File value", func() { + buf := bytes.NewBufferString("target:") + s := make.NewParser(buf, nil) + + f, err := s.ParseFile() + + Expect(err).NotTo(HaveOccurred()) + Expect(f.Rules).NotTo(BeEmpty()) }) }) diff --git a/scanner.go b/scanner.go index eb9d654..6951de8 100644 --- a/scanner.go +++ b/scanner.go @@ -4,6 +4,7 @@ import ( "bufio" "bytes" "io" + "math" "github.com/unmango/go-make/token" ) @@ -24,6 +25,9 @@ func NewScanner(r io.Reader, file *token.File) *Scanner { scanner := bufio.NewScanner(r) scanner.Split(ScanTokens) + if file == nil { + file = token.NewFileSet().AddFile("", 1, math.MaxInt-2) + } s := &Scanner{ s: scanner, file: file, diff --git a/scanner_test.go b/scanner_test.go index d3ab0b2..22ad0c4 100644 --- a/scanner_test.go +++ b/scanner_test.go @@ -326,4 +326,14 @@ var _ = Describe("Scanner", func() { _, _, _ = s.Scan() Expect(s.Err()).To(MatchError("io error")) }) + + It("should support a nil *token.File value", func() { + buf := bytes.NewBufferString("target:") + s := make.NewScanner(buf, nil) + + pos, tok, lit := s.Scan() + Expect(tok).To(Equal(token.IDENT)) + Expect(lit).To(Equal("target")) + Expect(pos).To(Equal(token.Pos(1))) + }) }) diff --git a/token/position.go b/token/position.go index b2273ee..d6709a9 100644 --- a/token/position.go +++ b/token/position.go @@ -18,3 +18,5 @@ const NoPos = token.NoPos func PositionFor(file *File, p Pos) Position { return file.PositionFor(p, false) } + +var NewFileSet = token.NewFileSet