Move grammar generation to Style package

M2/Macaulay2/editors/Macaulay2Web/M2-symbols.ts.in

@@ -1 +1,3 @@
+// @M2BANNER@
+
 export default [@M2SYMBOLS@];

M2/Macaulay2/editors/Makefile.in

@@ -38,7 +38,8 @@ highlightjs/%: @srcdir@/highlightjs/% highlightjs
 	cp $< $@
 
 clean:
-	rm -rf @pre_emacsdir@/* atom emacs highlightjs prism pygments vim
+	rm -rf @pre_emacsdir@/* atom emacs highlightjs prism pygments vim \
+		Macaulay2Web textmate
 
 else
 clean:

M2/Macaulay2/editors/atom/macaulay2.cson.in

@@ -1,3 +1,5 @@
+## @M2BANNER@
+
 # If this is your first time writing a language grammar, check out:
 #   - https://flight-manual.atom.io/hacking-atom/sections/creating-a-grammar/
 #   - https://flight-manual.atom.io/hacking-atom/sections/creating-a-legacy-textmate-grammar/

M2/Macaulay2/editors/make-M2-symbols.m2

@@ -3,201 +3,35 @@
 -- as keywords, types, etc. and substituting them in grammar files for various
 -- editors and syntax highlighting engines. Grammar file templates are assumed
 -- to be located in the same directory as this script.
--- Currently:
---  - Emacs
---  - Atom & Linguist: https://github.com/Macaulay2/language-macaulay2
---  - Rouge
---  - Pygments
---  - highlight.js
-
--------------------------------------------------------------------------------
-exportFrom_Core "sortBy"
-
--- TODO: Move these two elsewhere:
-Function and Function := (f, g) -> s -> f s and g s
-Function or  Function := (f, g) -> s -> f s or  g s
-
-is := X -> (name, symb) -> instance(value symb, X)
-
-isAlpha        := s -> match("^[[:alpha:]]+$", s)
-isAlphaNumeric := s -> match("^[[:alnum:]]+$", s)
-
--- Things we want to highlight:
-isType     := is Type
-isKeyword  := is Keyword
-isFunction := is Function
-isConst    := (name, symb) -> (isAlphaNumeric name
-    and not (isFunction or isType or isKeyword) (name, symb)
-    and (symb === symbol null or value symb =!= null))
-
-okay := method()
-okay(String, Keyword) :=
-okay(String, Symbol)  := (name, pkg) -> length name > 1 and isAlphaNumeric name
-
--------------------------------------------------------------------------------
--- Get a list of all symbols visible just after loading preloaded packages
-allPkgNames := separate(" ", version#"packages") | {"Core"}
-loadedPkgNames := Core#"preloaded packages" | {"Core", "Text", "Parsing", "SimpleDoc"}
-symbols := unique (sortBy ascii @@ first) join(
-    apply(allPkgNames, pkgname -> (pkgname, symbol Core)),
-    flatten apply(loadedPkgNames, pkgname -> (
-	    pkg := needsPackage pkgname;
-	    select(pairs pkg.Dictionary, okay))))
-
-if length symbols < 1500 then error "expected more entries for M2-symbols"
-
--- Check for invalid symbols
-bad := select(symbols, (name, symb) -> not okay(name, symb))
-if #bad > 0 then error(
-    "encountered symbol(s) that are not alphanumeric or have less than 2 characters:",
-    concatenate apply(bad, (name, symb) ->
-	{"\n\t", -* TODO: symbolLocation name, ": here is the first use of ", *- toString name}))
-
--------------------------------------------------------------------------------
--- Put the symbols into bins
-SYMBOLS   = first \ select(symbols, (name, symb) -> isAlphaNumeric name)
-KEYWORDS  = first \ select(symbols, isKeyword)
-DATATYPES = first \ select(symbols, isType)
-FUNCTIONS = first \ select(symbols, isFunction)
-CONSTANTS = first \ select(symbols, isConst)
-CONSTANTS = CONSTANTS | {"Node", "Item", "Example", "CannedExample", "Pre", "Code", "Tree", "Synopsis"} -- SimpleDoc words
-CONSTANTS = sort CONSTANTS
-STRINGS   = format "///\\\\(/?/?[^/]\\\\|\\\\(//\\\\)*////[^/]\\\\)*\\\\(//\\\\)*///"
-
 -------------------------------------------------------------------------------
--- Substitute symbols, keywords, types, functions, and constants
 
--- This banner is added to the top of generated grammars
-banner := "Auto-generated for Macaulay2-@M2VERSION@. Do not modify this file manually."
-
-symbolsForVim = template -> (
-    output := concatenate("\"\" ", banner, newline, newline, template);
-    output = replace("@M2VERSION@",   version#"VERSION",      output);
-    output = replace("@M2SYMBOLS@",   demark(" ", SYMBOLS),   output);
-    output = replace("@M2KEYWORDS@",  demark(" ", KEYWORDS),  output);
-    output = replace("@M2DATATYPES@", demark(" ", DATATYPES), output);
-    output = replace("@M2FUNCTIONS@", demark(" ", FUNCTIONS), output);
-    output = replace("@M2CONSTANTS@", demark(" ", CONSTANTS), output);
-    output = replace("@M2STRINGS@",               STRINGS,    output);
-    output)
-
-symbolsForEmacs = template -> (
-    output := concatenate(";; ", banner, newline, newline, template);
-    output = replace("@M2VERSION@",   version#"VERSION",               output);
-    output = replace("@M2SYMBOLS@",   demark(" ", format \ SYMBOLS),   output);
-    output = replace("@M2KEYWORDS@",  demark(" ", format \ KEYWORDS),  output);
-    output = replace("@M2DATATYPES@", demark(" ", format \ DATATYPES), output);
-    output = replace("@M2FUNCTIONS@", demark(" ", format \ FUNCTIONS), output);
-    output = replace("@M2CONSTANTS@", demark(" ", format \ CONSTANTS), output);
-    output = replace("@M2STRINGS@",                        STRINGS,    output);
-    output)
-
-symbolsForAtom = template -> (
-    output := concatenate("## ", banner, newline, newline, template);
-    output = replace("@M2VERSION@",   version#"VERSION",      output);
-    --output = replace("@M2SYMBOLS@",   demark("|", SYMBOLS),   output);
-    output = replace("@M2KEYWORDS@",  demark("|", KEYWORDS),  output);
-    output = replace("@M2DATATYPES@", demark("|", DATATYPES), output);
-    output = replace("@M2FUNCTIONS@", demark("|", FUNCTIONS), output);
-    output = replace("@M2CONSTANTS@", demark("|", CONSTANTS), output);
-    output = replace("@M2STRINGS@",               STRINGS,    output);
-    output)
-
-symbolsForPrism = template -> (
-    output := concatenate("// ", banner, newline, newline, template);
-    output = replace("@M2VERSION@",   version#"VERSION",      output);
-    output = replace("@M2SYMBOLS@",   demark(",", format \ SYMBOLS), output);
-    output = replace("@M2KEYWORDS@",  demark("|", KEYWORDS),  output);
-    output = replace("@M2DATATYPES@", demark("|", DATATYPES), output);
-    output = replace("@M2FUNCTIONS@", demark("|", FUNCTIONS), output);
-    output = replace("@M2CONSTANTS@", demark("|", CONSTANTS), output);
-    output = replace("@M2STRINGS@",               STRINGS,    output);
-    output)
-
-symbolsForRouge = template -> (
-    output := concatenate("## ", banner, newline, newline, template);
-    output = replace("@M2VERSION@",   version#"VERSION",      output);
-    --output = replace("@M2SYMBOLS@",   demark("|", SYMBOLS),   output);
-    output = replace("@M2KEYWORDS@",  demark(" ", KEYWORDS),  output);
-    output = replace("@M2DATATYPES@", demark("|", DATATYPES), output);
-    output = replace("@M2FUNCTIONS@", demark("|", FUNCTIONS), output);
-    output = replace("@M2CONSTANTS@", demark("|", CONSTANTS), output);
-    output = replace("@M2STRINGS@",               STRINGS,    output);
-    output)
-
-pygmentsformat = symlist -> demark("," | newline | "    ", format \ symlist)
-symbolsForPygments = template -> (
-    output := replace("@M2BANNER@", "# " | banner, template);
-    output = replace("@M2VERSION@",   version#"VERSION",        output);
-    output = replace("@M2KEYWORDS@",  pygmentsformat KEYWORDS,  output);
-    output = replace("@M2DATATYPES@", pygmentsformat DATATYPES, output);
-    output = replace("@M2FUNCTIONS@", pygmentsformat FUNCTIONS, output);
-    output = replace("@M2CONSTANTS@", pygmentsformat CONSTANTS, output);
-    output = replace("@M2STRINGS@",                  STRINGS,   output);
-    output)
-
-hljsformat = symlist -> demark("," | newline | "	", format \ symlist)
-symbolsForHighlightJS = template -> (
-    output := replace("@M2BANNER@",   banner,               template);
-    output = replace("@M2VERSION@",   version#"VERSION",    output);
-    output = replace("@M2KEYWORDS@",  hljsformat KEYWORDS,  output);
-    output = replace("@M2DATATYPES@", hljsformat DATATYPES, output);
-    output = replace("@M2FUNCTIONS@", hljsformat FUNCTIONS, output);
-    output = replace("@M2CONSTANTS@", hljsformat CONSTANTS, output);
-    output)
-
-symbolsForTextMate = template -> (
-    output = replace("@M2KEYWORDS@",  demark("|", KEYWORDS),  template);
-    output = replace("@M2DATATYPES@", demark("|", DATATYPES), output);
-    output = replace("@M2FUNCTIONS@", demark("|", FUNCTIONS), output);
-    output = replace("@M2CONSTANTS@", demark("|", CONSTANTS), output);
-    output)
-
-symbolsForMacaulay2Web = template -> (
-    output := concatenate("// ", banner, newline, newline, template);
-    output = replace("@M2VERSION@",   version#"VERSION",    output);
-    output = replace("@M2SYMBOLS@",   demark(",", format \ SYMBOLS), output);
-    output)
-
--------------------------------------------------------------------------------
--- Generate syntax files from templates in the same directory
-
-generateGrammar := (grammarFile, grammarFunction) -> (
-    template := currentFileDirectory | grammarFile | ".in";
-    if fileExists template then (
-        stdio << "-- Generating " << grammarFile << endl;
-        directory := replace("/[^/].*$", "", grammarFile);
-        if not isDirectory directory then makeDirectory directory;
-        grammarFile << grammarFunction get(template) << close)
-    else stderr << "Skipping generation of " << grammarFile << " as it does not exist." << endl;)
+needsPackage "Style"
 
 -- Emacs: Write M2-symbols.el
-generateGrammar("emacs/M2-symbols.el", symbolsForEmacs)
+generateGrammar("emacs/M2-symbols.el", x -> demark(" ", format \ x))
 
 -- Atom & Linguist: Write macaulay2.cson
-generateGrammar("atom/macaulay2.cson", symbolsForAtom);
+generateGrammar("atom/macaulay2.cson", x -> demark("|", x))
 
 -- Prism: Write macaulay2.js
-generateGrammar("prism/macaulay2.js", symbolsForPrism);
+generateGrammar("prism/macaulay2.js", x -> demark("|", x))
 
 -- Vim: Write m2.vim.syntax and m2.vim.dict
-generateGrammar("vim/m2.vim.syntax", symbolsForVim);
-generateGrammar("vim/m2.vim.dict", symbolsForVim); -- TODO: is this necessary?
-
--- Rouge: Write macaulay2.rb
---generateGrammar("rouge/macaulay2.rb", symbolsForRouge);
+generateGrammar("vim/m2.vim.syntax", x -> demark(" ", x))
+generateGrammar("vim/m2.vim.dict", x -> demark(" ", x)) -- TODO: is this necessary?
 
 -- Pygments: Write macaulay2.py
-generateGrammar("pygments/macaulay2.py", symbolsForPygments)
+generateGrammar("pygments/macaulay2.py",
+    x -> demark("," | newline | "    ", format \ x))
 
 -- highlight.js: Write macaulay2.js
-generateGrammar("highlightjs/macaulay2.js", symbolsForHighlightJS)
+generateGrammar("highlightjs/macaulay2.js",
+    x -> demark("," | newline | "	", format \ x))
 
-generateGrammar("textmate/macaulay2.tmLanguage.json", symbolsForTextMate)
+generateGrammar("textmate/macaulay2.tmLanguage.json", x -> demark("|", x))
 
 -- Macaulay2Web: Write M2-symbols.ts
-generateGrammar("Macaulay2Web/M2-symbols.ts", symbolsForMacaulay2Web)
-
+generateGrammar("Macaulay2Web/M2-symbols.ts", x -> demark(",", format \ x))
 
 -- Local Variables:
 -- compile-command: "make -C $M2BUILDDIR/Macaulay2/emacs M2-symbols "

M2/Macaulay2/editors/prism/macaulay2.js.in

@@ -1,3 +1,5 @@
+// @M2BANNER@
+
 Prism.languages.m2 =
 Prism.languages.macaulay2 = {
     'comment': [

M2/Macaulay2/editors/vim/m2.vim.dict.in

@@ -1,3 +1,5 @@
+"" @M2BANNER@
+
 " Vim dictionary file
 " Language: Macaulay2
 

M2/Macaulay2/editors/vim/m2.vim.syntax.in

@@ -1,3 +1,5 @@
+"" @M2BANNER@
+
 " Vim syntax file
 " Language: Macaulay2
 

M2/Macaulay2/m2/integers.m2

@@ -92,6 +92,11 @@ ZZ & ZZ := ZZ => lookup(symbol &, ZZ, ZZ)
 ZZ ^^ ZZ := bitxorfun
 Boolean xor Boolean := (x, y) -> x and not y or not x and y
 
+Function and Function := (f, g) -> s -> f s and g s
+Function or  Function := (f, g) -> s -> f s or  g s
+Function xor Function := (f, g) -> s -> f s xor g s
+not Function := f -> s -> not f s
+
 ZZ~ := bitnotfun
 
 changeBase = method()

M2/Macaulay2/packages/Macaulay2Doc/operators.m2

@@ -250,7 +250,7 @@ document {
 
 -- TODO: implement or/and for ZZ
 document {
-     Key => "or",
+     Key => {"or", (symbol or, Function, Function)},
      Headline => "disjunction",
      TT "t or u", " -- returns true if ", TT "t", " is true or ", TT "u", "
      is true.",
@@ -260,7 +260,7 @@ document {
      }
 
 document {
-     Key => "and",
+     Key => {"and", (symbol and, Function, Function)},
      Headline => "conjunction",
      TT "t and u", " -- returns true if ", TT "t", " is true and ", TT "u", "
      is true.",
@@ -270,7 +270,7 @@ document {
      }
 
 document {
-     Key => "not",
+     Key => {"not", (symbol not, Function)},
      Headline => "negation",
      TT "not x", " -- yields the negation of x, which must be true or false.",
      SeeAlso =>{ "and", "or" }
@@ -280,6 +280,7 @@ doc ///
   Key
     symbol xor
     (symbol xor, Boolean, Boolean)
+    (symbol xor, Function, Function)
   Headline
     exclusive disjunction
   Usage

M2/Macaulay2/packages/SimpleDoc.m2

@@ -66,6 +66,8 @@ applySplit = (functionTable, textlines) -> apply(splitByIndent(textlines, false)
 	functionTable#key(textlines_{s+1..e}, getLinenum textlines#s)))
 
 -- Mapping tables for evaluating docstring keywords
+-- When adding a new keyword that isn't already an exported symbol,
+-- add it to the list of constants for syntax highlighting in Style.m2
 NodeFunctions = new HashTable from {
     "Node"            => (textlines, keylinenum) -> new Node from nodeCheck(applySplit(NodeFunctions, textlines), keylinenum),
     "Key"             => (textlines, keylinenum) -> Key             => getKeys(textlines, keylinenum),