Merge pull request #2275 from hzeller/feature-20241003-print-raw-tokens

Give newline token symbolic name; print raw tokens c-escaped.

Author: hzeller

common/text/BUILD

@@ -26,6 +26,7 @@ cc_library(
         "//common/util:iterator-range",
         "//common/util:logging",
         "//common/util:range",
+        "@com_google_absl//absl/strings",
         "@com_google_absl//absl/strings:string_view",
     ],
 )

common/text/token_info.cc

@@ -20,6 +20,7 @@
 #include <string>
 #include <vector>
 
+#include "absl/strings/escaping.h"
 #include "absl/strings/string_view.h"
 #include "common/strings/rebase.h"
 #include "common/text/constants.h"
@@ -53,14 +54,15 @@ std::ostream &TokenInfo::ToStream(std::ostream &output_stream,
   output_stream << "(#";
   context.token_enum_translator(output_stream, token_enum_);
   output_stream << " @" << left(context.base) << '-' << right(context.base)
-                << ": \"" << text_ << "\")";
+                << ": \"" << absl::CEscape(text_) << "\")";
   const auto dist = std::distance(context.base.end(), text_.end());
   CHECK(IsSubRange(text_, context.base)) << "text.end() is off by " << dist;
   return output_stream;
 }
 
 std::ostream &TokenInfo::ToStream(std::ostream &output_stream) const {
-  return output_stream << "(#" << token_enum_ << ": \"" << text_ << "\")";
+  return output_stream << "(#" << token_enum_ << ": \"" << absl::CEscape(text_)
+                       << "\")";
 }
 
 std::string TokenInfo::ToString(const Context &context) const {

verilog/parser/verilog.y

@@ -667,7 +667,7 @@ is not locally defined, so the grammar here uses only generic identifiers.
 %token TK_COMMENT_BLOCK "/∗comment∗/"
 %token TK_EOL_COMMENT "// end of line comment"
 %token TK_SPACE "<<space>>"  /* includes tabs */
-%token TK_NEWLINE "<<\\n>>"
+%token TK_NEWLINE "<<newline>>"
 %token TK_LINE_CONT "<<\\line-cont>>"
 %token TK_ATTRIBUTE "(*attribute*)"
 
@@ -3518,12 +3518,12 @@ instantiation_base
     { $$ = MakeInstantiationBase($1, $2); }
   /*
    * TODO: support mixed anonymous declarations
-   * 
+   *
    * This production rule was commented out because it caused
    * verible-verilog-syntax to crash for some inputs. It may be necessary to
    * re-enable it in the future to support declarations that mix anonymous and
    * named instances.
-   * 
+   *
    * For more details, see https://github.com/chipsalliance/verible/issues/2181
    */
   // | reference call_base ',' gate_instance_or_register_variable_list

verilog/tools/syntax/verilog_syntax_test.sh

@@ -307,11 +307,9 @@ All lexed tokens:
 (#"<<space>>" @6-7: " ")
 (#SymbolIdentifier @7-9: "mm")
 (#';' @9-10: ";")
-(#"<<\\\\n>>" @10-11: "
-")
+(#"<<newline>>" @10-11: "\\n")
 (#"endmodule" @11-20: "endmodule")
-(#"<<\\\\n>>" @20-21: "
-")
+(#"<<newline>>" @20-21: "\\n")
 (#"end of file" @21-21: "")
 EOF