Remove KeywordParser

languageServer/src/main/scala/core/bigrammar/BiGrammarWriter.scala

@@ -54,7 +54,7 @@ trait BiGrammarWriter {
     if (count == value.length)
       Keyword(value, reserved)
     else if (count == 0)
-      Delimiter(value)
+      BiLiteral(value)
     else
       throw new RuntimeException(s"Can't implicitly convert $value to BiGrammar because of mixed character types")
   }

languageServer/src/main/scala/core/bigrammar/PrintBiGrammar.scala

@@ -40,7 +40,7 @@ object PrintBiGrammar {
     case regex: RegexGrammar => s"Regex(${regex.regex})"
     case keyword: Keyword => keyword.value
     case _: Identifier => "Identifier"
-    case delimiter: Delimiter => delimiter.value
+    case delimiter: BiLiteral => delimiter.value
     case ValueGrammar(value) => if (value == null) "null" else value.toString
     case BiFailure(message) => message
     case labelled: Labelled => grammarKeyToName(labelled.name)

languageServer/src/main/scala/core/bigrammar/grammars/Delimiter.scala → languageServer/src/main/scala/core/bigrammar/grammars/BiLiteral.scala

@@ -3,7 +3,12 @@ package core.bigrammar.grammars
 import core.bigrammar.BiGrammarToParser
 import core.parsers.editorParsers.History
 
-case class Delimiter(value: String, penalty: Double = History.missingInputPenalty, allowDrop: Boolean = true) extends StringGrammar {
+case class BiLiteral(value: String,
+                     penalty: Double = History.missingInputPenalty,
+                     allowDrop: Boolean = true,
+                     verifyWhenPrinting: Boolean = false)
+  extends StringGrammar(verifyWhenPrinting) {
+
   if (value.length == 0)
     throw new RuntimeException("value must have non-zero length")
 

languageServer/src/main/scala/core/bigrammar/grammars/Keyword.scala

@@ -9,9 +9,6 @@ case class Keyword(var value: String, reserved: Boolean = true, verifyWhenPrinti
     throw new RuntimeException("value must have non-zero length")
 
   override def getParserBuilder(keywords: scala.collection.Set[String]): Self[String] = {
-    if (reserved)
-      literalOrKeyword(value)
-    else
-      literal(value)
+    literal(value)
   }
 }

languageServer/src/main/scala/core/bigrammar/printer/BiGrammarToPrinter.scala

@@ -31,7 +31,7 @@ class BiGrammarToPrinter {
     printerCache.getOrElseUpdate(grammar, {
       val result: NodePrinter = grammar match {
         case choice: BiChoice => new OrPrinter(toPrinterCached(choice.left), toPrinterCached(choice.right))
-        case delimiter: Delimiter => _ => succeed(delimiter.value)
+        case delimiter: BiLiteral => _ => succeed(delimiter.value)
         case Keyword(keyword, _, verify) => value =>
           if (!verify || value.value == keyword)
             succeed(keyword)

languageServer/src/main/scala/core/bigrammar/textMate/BiGrammarToTextMate.scala

@@ -40,7 +40,7 @@ object BiGrammarToTextMate {
         case map: MapGrammar => recurse(map.inner)
         case map: MapGrammarWithMap => recurse(map.inner)
         case labelled: Labelled => recurse(labelled.inner)
-        case delimiter: Delimiter => Some(escapeRegex(delimiter.value))
+        case delimiter: BiLiteral => Some(escapeRegex(delimiter.value))
         case _: Identifier => Some(escapeRegex(BiGrammarToParser.identifierRegex.regex))
         case ParseWhiteSpace => Some(ParseWhiteSpace.regex.regex)
         case keyword: Keyword => Some(escapeRegex(keyword.value))
@@ -137,7 +137,7 @@ object BiGrammarToTextMate {
       node => if (node.isInstanceOf[Colorize]) GraphBasics.SkipChildren else GraphBasics.Continue )
 
     val typedPatterns: Seq[Match] = reachables.collect({
-      case delimiter: Delimiter =>
+      case delimiter: BiLiteral =>
         Match("keyword.operator", escapeRegex(delimiter.value).r)
       case keyword: Keyword /*if keyword.reserved*/ =>
         Match("keyword.control", ("\\b" + escapeRegex(keyword.value) + "\\b").r)

languageServer/src/main/scala/core/bigrammar/textMate/TextMateDelta.scala

@@ -1,6 +1,6 @@
 package core.bigrammar.textMate
 
-import core.bigrammar.grammars.Delimiter
+import core.bigrammar.grammars.BiLiteral
 import core.deltas.DeltaWithGrammar
 import core.deltas.grammars.LanguageGrammars
 import core.language.Language
@@ -14,12 +14,12 @@ object TextMateDelta extends DeltaWithGrammar {
   override def transformGrammars(grammars: LanguageGrammars, language: Language): Unit = {
     import grammars._
     val objectLiteral = find(JsonObjectLiteralDelta.Shape)
-    objectLiteral.find(p => p.value == Delimiter(":")).get.set(printSpace ~> Delimiter("="))
-    objectLiteral.find(p => p.value == Delimiter(",")).get.set(Delimiter(";"))
+    objectLiteral.find(p => p.value == BiLiteral(":")).get.set(printSpace ~> BiLiteral("="))
+    objectLiteral.find(p => p.value == BiLiteral(",")).get.set(BiLiteral(";"))
 
     val arrayLiteral = find(ArrayLiteralDelta.Shape)
-    arrayLiteral.find(p => p.value == Delimiter("[")).get.set(Delimiter("("))
-    arrayLiteral.find(p => p.value == Delimiter("]")).get.set(Delimiter(")"))
+    arrayLiteral.find(p => p.value == BiLiteral("[")).get.set(BiLiteral("("))
+    arrayLiteral.find(p => p.value == BiLiteral("]")).get.set(BiLiteral(")"))
   }
 
   override def description = "Introduces TextMate syntax"

languageServer/src/main/scala/core/parsers/strings/StringParserWriter.scala

@@ -66,13 +66,8 @@ trait StringParserWriter extends SequenceParserWriter {
 
   implicit def literalToExtensions(value: String): SequenceParserExtensions[String] = Literal(value)
 
-  implicit def stringToLiteralOrKeyword(value: String): Self[String] = {
-    literalOrKeyword(value)
-  }
-
-  def literalOrKeyword(value: String, allowDrop: Boolean = true): Self[String] = {
-    val isKeyword = identifierRegex.findFirstIn(value).contains(value)
-    if (isKeyword) KeywordParser(value) else literal(value, allowDrop = allowDrop)
+  implicit def stringToLiteral(value: String): Self[String] = {
+    literal(value)
   }
 
   def literal(value: String, penalty: Double = History.missingInputPenalty,
@@ -109,43 +104,6 @@ trait StringParserWriter extends SequenceParserWriter {
     override def getMustConsume(cache: ConsumeCache) = value.nonEmpty
   }
 
-  /**
-    * Don't wrap KeywordParser in a Drop. Since it wraps identifier, it already has a drop.
-    * What's the point of letting KeywordParser use identifier instead of Literal?
-    * Using Literal would be much simpler.
-    * The situation is that identifier and keyword parser can overlap, but identifier has a filter on keywords
-    */
-  case class KeywordParser(value: String) extends ParserBuilderBase[String] with ParserWrapper[String] {
-    override def getParser(recursive: GetParser): Parser[String] = {
-      val parseIdentifier = recursive(identifier)
-      (input, state) => {
-        parseIdentifier(input, state).mapReady(ready => {
-          if (ready.resultOption.contains(value)) {
-            ready
-          } else {
-            var parsed = 0
-            val parsedIdentifier = ready.resultOption.getOrElse("")
-            while(parsed < parsedIdentifier.length &&  parsed < value.length &&
-              parsedIdentifier.charAt(parsed) == value.charAt(parsed)) {
-              parsed += 1
-            }
-            val insertFix = value.drop(parsed)
-            val reached = input.safeDrop(parsed)
-            val error =
-              if (ready.remainder == input)
-                new MissingInput(input, value, insertFix, History.missingInputPenalty)
-              else
-                MissingInput(input, ready.remainder, value, insertFix, History.missingInputPenalty)
-
-            ReadyParseResult(Some(value), reached, History.error(error))
-          }
-        }, uniform = false)
-      }
-    }
-
-    override def original: Self[String] = identifier
-  }
-
   trait NextCharError extends ParseError[Input] {
     def to: Input = if (this.from.atEnd) this.from else this.from.drop(1)
     def range = SourceRange(from.position, to.position)

languageServer/src/main/scala/deltas/expression/ArrayLiteralDelta.scala

@@ -1,6 +1,6 @@
 package deltas.expression
 
-import core.bigrammar.grammars.Delimiter
+import core.bigrammar.grammars.BiLiteral
 import core.deltas.DeltaWithGrammar
 import core.deltas.grammars.LanguageGrammars
 import core.deltas.path.NodePath
@@ -27,7 +27,7 @@ object ArrayLiteralDelta extends DeltaWithGrammar with ExpressionInstance {
     import _grammars._
 
     val expression = find(ExpressionDelta.FirstPrecedenceGrammar)
-    val array = Delimiter("[", History.missingInputPenalty * 2) ~ expression.manySeparated(",").as(Members) ~ "]"
+    val array = BiLiteral("[", History.missingInputPenalty * 2) ~ expression.manySeparated(",").as(Members) ~ "]"
     val grammar = array.asLabelledNode(Shape)
     expression.addAlternative(grammar)
   }

languageServer/src/main/scala/deltas/json/JsonObjectLiteralDelta.scala

@@ -37,7 +37,7 @@ object JsonObjectLiteralDelta extends DeltaWithGrammar with ExpressionInstance w
 
     val member = (Colorize(create(MemberKey, keyGrammar), "string.quoted.double") ~< ":") ~~ expressionGrammar.as(MemberValue) asNode MemberShape
     val optionalTrailingComma = Parse(Keyword(",") | value(Unit))
-    val inner = Delimiter("{", History.missingInputPenalty * 2) %> (member.manySeparatedVertical(",").as(Members) ~< optionalTrailingComma).indent() %< "}"
+    val inner = BiLiteral("{", History.missingInputPenalty * 2) %> (member.manySeparatedVertical(",").as(Members) ~< optionalTrailingComma).indent() %< "}"
 
     val grammar = inner.asLabelledNode(Shape)
     expressionGrammar.addAlternative(grammar)

languageServer/src/main/scala/deltas/json/SingleQuotedStringLiteralDelta.scala

@@ -1,6 +1,6 @@
 package deltas.json
 
-import core.bigrammar.grammars.{As, Colorize, Delimiter, RegexGrammar}
+import core.bigrammar.grammars.{As, Colorize, BiLiteral, RegexGrammar}
 import core.deltas.grammars.LanguageGrammars
 import core.deltas.{Contract, DeltaWithGrammar}
 import core.language.Language
@@ -19,7 +19,7 @@ object SingleQuotedStringLiteralDelta extends DeltaWithGrammar {
     val inner = {
       import core.bigrammar.DefaultBiGrammarWriter._
       val withoutColor = JsonStringLiteralDelta.dropPrefix(grammars,
-        grammars.regexGrammar("""'[^']*""".r, "single quote string literal"), JsonStringLiteralDelta.Value, "'") ~< Delimiter("'")
+        grammars.regexGrammar("""'[^']*""".r, "single quote string literal"), JsonStringLiteralDelta.Value, "'") ~< BiLiteral("'")
       Colorize(withoutColor, "string.quoted.single")
     }
     import grammars._

languageServer/src/main/scala/deltas/yaml/PlainScalarDelta.scala

@@ -1,7 +1,7 @@
 package deltas.yaml
 
 import core.bigrammar.BiGrammar
-import core.bigrammar.grammars.{BiSequence, Delimiter, RegexGrammar, SequenceBijective}
+import core.bigrammar.grammars.{BiSequence, BiLiteral, RegexGrammar, SequenceBijective}
 import core.deltas.DeltaWithGrammar
 import core.deltas.grammars.LanguageGrammars
 import core.language.Language
@@ -35,7 +35,7 @@ object PlainScalarDelta extends DeltaWithGrammar {
 
     val plainStyleSingleLineString: BiGrammar = nsPlainSafe
     val plainStyleMultiLineString: BiGrammar = {
-      val lineSeparator = new BiSequence(Delimiter("\n", allowDrop = false), _grammars.trivia, BiSequence.ignoreLeft, true)
+      val lineSeparator = new BiSequence(BiLiteral("\n", allowDrop = false), _grammars.trivia, BiSequence.ignoreLeft, true)
       val firstLine = new BiSequence(nsPlainSafe, lineSeparator, BiSequence.ignoreRight, false)
       val followingLine = CheckIndentationGrammar.equal(nsPlainSafe)
       val otherLines = CheckIndentationGrammar.greaterThan(new WithIndentationGrammar(followingLine.someSeparated(lineSeparator)))

languageServer/src/test/scala/core/parsers/LeftRecursionTest.scala

@@ -152,12 +152,12 @@ class LeftRecursionTest extends FunSuite with CommonStringReaderParser
     val variable = new Lazy(identifier.map(x => x), "variable")
     lazy val lastExpression = new Lazy(Fail(None, "last", 1) | variable | call, "lastExpression")
     lazy val relationalInner: Self[Any] = new Lazy(lastExpression, "expression")
-    lazy val equality: Self[Any] = new Lazy(relational.map(x => x) ~ literalOrKeyword("==") ~ relationalInner.map(x => x), "equality")
+    lazy val equality: Self[Any] = new Lazy(relational.map(x => x) ~ literal("==") ~ relationalInner.map(x => x), "equality")
     lazy val relational: Self[Any] = new Lazy(relationalInner | equality, "relational")
     lazy val assignmentTarget: Self[Any] = new Lazy(Fail(None, "fail", 1), "assignmentTarget")
-    lazy val simpleAssignment: Self[Any] = new Lazy(assignmentTarget.map(x => x) ~ literalOrKeyword("=") ~ expression.map(x => x), "equality")
+    lazy val simpleAssignment: Self[Any] = new Lazy(assignmentTarget.map(x => x) ~ literal("=") ~ expression.map(x => x), "equality")
     lazy val assignment: Self[Any] = new Lazy(relational | simpleAssignment, "assignemnt")
-    lazy val memberSelector = new Lazy(expression.map(x => x) ~ literalOrKeyword(".") ~ identifier.map(x => x), "member selector")
+    lazy val memberSelector = new Lazy(expression.map(x => x) ~ literal(".") ~ identifier.map(x => x), "member selector")
     lazy val callCallee = new Lazy(variable | memberSelector, "callCallee")
     lazy val call = new Lazy(callCallee.map(x => x) ~ "(" ~ expression.manySeparated(",", "parameter").map(x => x) ~ ")", "call")
     //val language = new LanguageTest(TestLanguageBuilder.buildWithParser(Seq(ClearPhases, ExpressionAsRoot) ++ JavaLanguage.fields ++ JavaLanguage.method))

languageServer/src/test/scala/core/parsers/YamlTest.scala

@@ -126,7 +126,7 @@ class YamlTest extends FunSuite
 
   lazy val parseBlockMapping: Self[YamlExpression] = {
     val member = new WithContext(_ =>
-      BlockKey, parseFlowValue) ~< literalOrKeyword(":") ~ greaterThan(parseValue)
+      BlockKey, parseFlowValue) ~< literal(":") ~ greaterThan(parseValue)
     alignedList(member).map(values => {
       Object(values.toMap)
     })
@@ -138,7 +138,7 @@ class YamlTest extends FunSuite
   }
 
   lazy val parseArray: Self[YamlExpression] = {
-    val element = literalOrKeyword("- ") ~> greaterThan(parseValue)
+    val element = literal("- ") ~> greaterThan(parseValue)
     alignedList(element).map(elements => Array(elements))
   }
 

playground/src/main/scala/application/compilerCockpit/TokenMakerFromGrammar.scala

@@ -28,9 +28,9 @@ class TokenMakerFromGrammar(grammar: BiGrammar) extends AbstractTokenMaker {
     val tokenParsers: Set[BiGrammarToParser.Self[MyToken]] = reachables.collect({
       case keyword: Keyword if keyword.reserved =>
         keywords.add(keyword.value)
-        literalOrKeyword(keyword.value, allowDrop = false) ^^ (s => MyToken(TokenTypes.RESERVED_WORD, s))
-      case delimiter: Delimiter =>
-        literalOrKeyword(delimiter.value, allowDrop = false) ^^ (s => MyToken(TokenTypes.SEPARATOR, s))
+        literal(keyword.value, allowDrop = false) ^^ (s => MyToken(TokenTypes.RESERVED_WORD, s))
+      case biLiteral: BiLiteral =>
+        literal(biLiteral.value, allowDrop = false) ^^ (s => MyToken(TokenTypes.SEPARATOR, s))
       case identifier: Identifier => identifier.getParserBuilder(keywords) ^^ (s => MyToken(TokenTypes.IDENTIFIER, s))
       case NumberGrammar => wholeNumber ^^ (s => MyToken(TokenTypes.LITERAL_NUMBER_DECIMAL_INT, s)) //TODO should support other numbers as well.
       case StringLiteral =>