Basic parsing (#9)

* Start actual parsing

* Remove walk export because it is Go specific

* Add method to append targets

A new method has been added to the TargetList struct. This method allows appending a target to the existing list of targets in an efficient manner.

* Refactor parser and update tests

The Parser has been refactored for improved readability and efficiency. The NewParser function now directly initializes the Scanner within the Parser struct, eliminating an unnecessary variable. The parseRule function has been streamlined to handle target parsing inline, removing the need for a separate parseTargets function. Error handling is also improved with early returns on error conditions.

Corresponding changes have been made in parser_test.go to reflect these modifications in the Parser structure and behavior.

* Added Codecov validation and configuration

Introduced a new makefile target 'validate_codecov' to validate the codecov.yml file. Also, added a new codecov.yml file with project coverage settings - setting the target coverage at 75% and threshold at 3%.

* Enhanced parsing functionality

Added new methods to the Parser struct in parser.go for improved parsing of rules, prerequisites and recipes. Also added corresponding tests in parser_test.go to ensure correct functionality. The ast.go file was updated with a method to append prerequisites to PreReqList.

* Added default file handling in parser and scanner

In this update, we've added a check for nil files in both the Parser and Scanner constructors. If a nil file is passed, a new one is created with maximum integer size. Additionally, we've exposed the NewFileSet function from the token package for external use.

* Updated README with new usage instructions

The README has been updated to reflect changes in the usage of the make.Parser, make.Scanner, and make.ScanTokens. The previous scanning utilities section has been replaced with a more detailed explanation on how to use these functions. Additionally, redundant future plans have been removed from the document.

* Updated AST test and added new tests

- Corrected the expected position after final recipe in AST test
- Added a new test to check if given target is appended correctly
- Fixed typo in PreReqList description
- Added a new test to verify if given prereq is appended as expected
- Extended Parser and Scanner tests to support nil *token.File value

* Updated coverage targets in codecov config

Added new patch coverage target and threshold to the codecov configuration. The patch target is set at 60% and the threshold at 10%. This will help ensure that new code changes maintain a certain level of test coverage.

Makefile

@@ -21,6 +21,8 @@ tidy: go.sum
 test_all:
 	$(GINKGO) run -r ./
 
+validate_codecov: .make/validate_codecov
+
 cover: cover.profile
 	go tool cover -func=$<
 
@@ -57,3 +59,7 @@ bin/devctl: .versions/devctl
 .make/test: $(shell $(DEVCTL) list --go) | bin/ginkgo bin/devctl
 	$(GINKGO) run ${TEST_FLAGS} $(sort $(dir $?))
 	@touch $@
+
+.make/validate_codecov: codecov.yml
+	curl -X POST --data-binary @codecov.yml https://codecov.io/validate
+	@touch $@

README.md

@@ -4,7 +4,20 @@ Makefile parsing and utilities in Go
 
 ## Usage
 
-At present the scanning utilities are the most tested.
+### Reading
+
+The `make.Parser` is the primary way to read Makefiles.
+
+```go
+f := os.Open("Makefile")
+p := make.NewParser(f, nil)
+
+m, err := p.ParseFile()
+
+fmt.Println(m.Rules)
+```
+
+The more primitive `make.Scanner` and `make.ScanTokens` used by `make.Parser` can be used individually.
 
 Using `make.ScanTokens` with a `bufio.Scanner`
 
@@ -23,19 +36,15 @@ Using `make.Scanner`
 
 ```go
 f := os.Open("Makefile")
-s := make.NewScanner(f)
+s := make.NewScanner(f, nil)
 
-for s.Scan() {
-  s.Token() // The current token.Token i.e. token.SIMPLE_ASSIGN
-  s.Literal() // Literal tokens as a string i.e. "identifier"
+for pos, tok, lit := s.Scan(); tok != token.EOF {
+  fmt.Println(pos) // The position of tok
+  fmt.Println(tok) // The current token.Token i.e. token.SIMPLE_ASSIGN
+  fmt.Println(lit) // Literal tokens as a string i.e. "identifier"
 }
 
 if err := s.Err(); err != nil {
   fmt.Println(err)
 }
 ```
-
-## Future
-
-- `make.Parser`
-- `make.Parse(file)`

ast/ast.go

@@ -8,8 +8,6 @@ import (
 
 type Node = ast.Node
 
-var Walk = ast.Walk
-
 // A File represents text content interpreted as the make syntax.
 // Most commonly this is a Makefile, but could also be any file
 // understood by make, i.e. include-me.mk
@@ -79,6 +77,11 @@ type TargetList struct {
 	List []FileName
 }
 
+// Add appends target to t.List
+func (t *TargetList) Add(target FileName) {
+	t.List = append(t.List, target)
+}
+
 // Pos implements Node
 func (t *TargetList) Pos() token.Pos {
 	return t.List[0].Pos()
@@ -95,6 +98,11 @@ type PreReqList struct {
 	List []FileName
 }
 
+// Add appends prereq to p.List
+func (p *PreReqList) Add(prereq FileName) {
+	p.List = append(p.List, prereq)
+}
+
 // Pos implements Node
 func (p *PreReqList) Pos() token.Pos {
 	return p.List[0].Pos()

ast/ast_test.go

@@ -60,10 +60,10 @@ var _ = Describe("Ast", func() {
 		It("should return the position after the final recipe", func() {
 			c := &ast.Rule{Recipes: []*ast.Recipe{{
 				TokPos: token.Pos(420),
+				Text:   "some text",
 			}}}
 
-			// TODO: This is wrong, should be position after text
-			Expect(c.End()).To(Equal(token.Pos(420)))
+			Expect(c.End()).To(Equal(token.Pos(429)))
 		})
 	})
 
@@ -89,6 +89,17 @@ var _ = Describe("Ast", func() {
 
 			Expect(c.End()).To(Equal(token.Pos(423)))
 		})
+
+		It("should append the given target", func() {
+			c := &ast.TargetList{}
+			elem := &ast.LiteralFileName{Name: &ast.Ident{
+				NamePos: token.Pos(69),
+			}}
+
+			c.Add(elem)
+
+			Expect(c.List).To(ContainElement(elem))
+		})
 	})
 
 	Describe("PreReqList", func() {
@@ -102,7 +113,7 @@ var _ = Describe("Ast", func() {
 			Expect(c.Pos()).To(Equal(token.Pos(69)))
 		})
 
-		It("should return the position after the lat prereq", func() {
+		It("should return the position after the last prereq", func() {
 			c := &ast.PreReqList{List: []ast.FileName{
 				&ast.LiteralFileName{Name: &ast.Ident{NamePos: token.Pos(69)}},
 				&ast.LiteralFileName{Name: &ast.Ident{
@@ -113,6 +124,17 @@ var _ = Describe("Ast", func() {
 
 			Expect(c.End()).To(Equal(token.Pos(423)))
 		})
+
+		It("should append the given prereq", func() {
+			c := &ast.PreReqList{}
+			elem := &ast.LiteralFileName{Name: &ast.Ident{
+				NamePos: token.Pos(69),
+			}}
+
+			c.Add(elem)
+
+			Expect(c.List).To(ContainElement(elem))
+		})
 	})
 
 	Describe("LiteralFileName", func() {

codecov.yml

@@ -0,0 +1,9 @@
+coverage:
+  status:
+    project:
+      default:
+        target: 75%
+        threshold: 3%
+      patch:
+        target: 60%
+        threshold: 10%

parser.go

@@ -3,6 +3,8 @@ package make
 import (
 	"go/scanner"
 	"io"
+	"math"
+	"strings"
 
 	"github.com/unmango/go-make/ast"
 	"github.com/unmango/go-make/token"
@@ -16,12 +18,20 @@ type Parser struct {
 	pos token.Pos
 	tok token.Token // one token look-ahead
 	lit string      // token literal
+
+	recipePrefix token.Token
 }
 
 func NewParser(r io.Reader, file *token.File) *Parser {
+	if file == nil {
+		file = token.NewFileSet().AddFile("", 1, math.MaxInt-2)
+	}
+
 	p := &Parser{
 		s:    NewScanner(r, file),
 		file: file,
+
+		recipePrefix: token.TAB,
 	}
 	p.next()
 
@@ -38,21 +48,35 @@ func (p *Parser) ParseFile() (*ast.File, error) {
 	}
 }
 
+func (p *Parser) error(pos token.Pos, msg string) {
+	epos := p.file.Position(pos)
+	p.errors.Add(epos, msg)
+}
+
+func (p *Parser) errorExpected(pos token.Pos, msg string) {
+	msg = "expected " + msg
+	if p.pos == pos {
+		switch {
+		case p.tok.IsLiteral():
+			msg += ", found " + p.lit
+		default:
+			msg += ", found '" + p.tok.String() + "'"
+		}
+	}
+
+	p.error(pos, msg)
+}
+
 func (p *Parser) expect(tok token.Token) token.Pos {
 	pos := p.pos
 	if p.tok != tok {
-		p.error(pos, "expected '"+tok.String()+"'")
+		p.errorExpected(pos, "'"+tok.String()+"'")
 	}
 
 	p.next()
 	return pos
 }
 
-func (p *Parser) error(pos token.Pos, msg string) {
-	epos := p.file.Position(pos)
-	p.errors.Add(epos, msg)
-}
-
 func (p *Parser) next() {
 	p.pos, p.tok, p.lit = p.s.Scan()
 }
@@ -76,14 +100,12 @@ func (p *Parser) parseFile() *ast.File {
 }
 
 func (p *Parser) parseRule() *ast.Rule {
-	if p.tok != token.IDENT {
-		p.expect(token.IDENT)
-		return nil
-	}
-
-	var targets []ast.FileName
+	targets := new(ast.TargetList)
 	for p.tok != token.COLON && p.tok != token.EOF {
-		targets = append(targets, p.parseFileName())
+		targets.Add(p.parseFileName())
+	}
+	if p.errors.Len() > 0 {
+		return nil
 	}
 
 	var colon token.Pos
@@ -93,22 +115,72 @@ func (p *Parser) parseRule() *ast.Rule {
 	} else {
 		p.expect(token.COLON)
 	}
+	if p.errors.Len() > 0 {
+		return nil
+	}
+
+	prereqs := new(ast.PreReqList)
+	for p.tok != token.NEWLINE && p.tok != token.EOF {
+		prereqs.Add(p.parseFileName())
+	}
+	if p.errors.Len() > 0 {
+		return nil
+	}
+	if p.tok == token.NEWLINE {
+		p.next()
+	}
+
+	recipes := make([]*ast.Recipe, 0)
+	for p.isRecipePrefix() && p.tok != token.EOF {
+		recipes = append(recipes, p.parseRecipe())
+	}
+	if p.errors.Len() > 0 {
+		return nil
+	}
 
 	return &ast.Rule{
-		Targets: &ast.TargetList{
-			List: targets,
-		},
+		Targets: targets,
 		Colon:   colon,
 		Pipe:    token.NoPos,
 		Semi:    token.NoPos,
-		PreReqs: &ast.PreReqList{},
-		Recipes: []*ast.Recipe{},
+		PreReqs: prereqs,
+		Recipes: recipes,
+	}
+}
+
+func (p Parser) isRecipePrefix() bool {
+	return p.tok == p.recipePrefix
+}
+
+func (p *Parser) parseRecipe() *ast.Recipe {
+	if !p.isRecipePrefix() {
+		p.expect(p.recipePrefix)
+		return nil
+	}
+
+	tokPos := p.pos
+	b := &strings.Builder{}
+	p.next()
+	for p.tok != token.NEWLINE && p.tok != token.EOF {
+		b.WriteString(p.lit)
+		p.next()
+	}
+	if p.tok == token.NEWLINE {
+		p.next()
+	}
+
+	return &ast.Recipe{
+		Tok:    token.TAB,
+		TokPos: tokPos,
+		Text:   b.String(),
 	}
 }
 
 func (p *Parser) parseFileName() ast.FileName {
+	name := p.parseIdent()
+
 	return &ast.LiteralFileName{
-		Name: p.parseIdent(),
+		Name: name,
 	}
 }