Lexer improvements for highlighter

Author: mjdetullio

sonar-objective-c-plugin/src/main/java/org/sonar/objectivec/api/ObjectiveCPunctuator.java

@@ -89,6 +89,7 @@ public enum ObjectiveCPunctuator implements TokenType {
 
     MINUSLT("->"),
     MINUSLTSTAR("->*"),
+    DOT("."),
     DOTSTAR(".*");
 
     private final String value;

sonar-objective-c-plugin/src/main/java/org/sonar/objectivec/api/ObjectiveCTokenType.java

@@ -19,12 +19,19 @@
  */
 package org.sonar.objectivec.api;
 
+import com.google.common.collect.ImmutableList;
 import com.sonar.sslr.api.AstNode;
 import com.sonar.sslr.api.TokenType;
 
-public enum ObjectiveCTokenType implements TokenType {
+import java.util.List;
 
-    NUMERIC_LITERAL;
+public enum ObjectiveCTokenType implements TokenType {
+    CHARACTER_LITERAL,
+    DOUBLE_LITERAL,
+    FLOAT_LITERAL,
+    INTEGER_LITERAL,
+    LONG_LITERAL,
+    STRING_LITERAL;
 
     @Override
     public String getName() {
@@ -41,4 +48,7 @@ public boolean hasToBeSkippedFromAst(AstNode node) {
         return false;
     }
 
+    public static List numberLiterals() {
+        return ImmutableList.of(DOUBLE_LITERAL, FLOAT_LITERAL, INTEGER_LITERAL, LONG_LITERAL);
+    }
 }

sonar-objective-c-plugin/src/main/java/org/sonar/objectivec/highlighter/SourceFileOffsets.java

@@ -33,6 +33,7 @@
 import org.sonar.api.batch.fs.InputFile;
 import org.sonar.api.source.Highlightable;
 import org.sonar.objectivec.api.ObjectiveCKeyword;
+import org.sonar.objectivec.api.ObjectiveCTokenType;
 import org.sonar.squidbridge.SquidAstVisitor;
 
 import java.io.IOException;
@@ -100,11 +101,13 @@ public void visitNode(AstNode astNode) {
 
     @Override
     public void visitToken(Token token) {
+        // Use org.sonar.api.batch.sensor.highlighting.TypeOfText here?
+
         for (Trivia trivia : token.getTrivia()) {
             if (trivia.isComment()) {
                 Token triviaToken = trivia.getToken();
                 if (triviaToken.getValue().startsWith("/**")) {
-                    highlightToken(triviaToken, "jd");
+                    highlightToken(triviaToken, "j");
                 } else if (triviaToken.getValue().startsWith("/*")) {
                     highlightToken(triviaToken, "cppd");
                 } else {
@@ -116,6 +119,15 @@ public void visitToken(Token token) {
         if (token.getType() instanceof ObjectiveCKeyword) {
             highlightToken(token, "k");
         }
+
+        if (ObjectiveCTokenType.numberLiterals().contains(token.getType())) {
+            highlightToken(token, "c");
+        }
+
+        if (ObjectiveCTokenType.STRING_LITERAL.equals(token.getType())
+                || ObjectiveCTokenType.CHARACTER_LITERAL.equals(token.getType())) {
+            highlightToken(token, "s");
+        }
     }
 
     private void highlightToken(Token token, String typeOfText) {

sonar-objective-c-plugin/src/main/java/org/sonar/objectivec/highlighter/SyntaxHighlighterVisitor.java

@@ -0,0 +1,46 @@
+/*
+ * SonarQube Objective-C (Community) Plugin
+ * Copyright (C) 2012-2016 OCTO Technology, Backelite, and contributors
+ * mailto:sonarqube@googlegroups.com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.objectivec.lexer;
+
+import com.sonar.sslr.impl.Lexer;
+import org.sonar.sslr.channel.Channel;
+import org.sonar.sslr.channel.CodeReader;
+
+/**
+ * @author Sonar C++ Plugin (Community) authors
+ */
+public class BackslashChannel extends Channel<Lexer> {
+    @Override
+    public boolean consume(CodeReader code, Lexer output) {
+        char ch = (char) code.peek();
+
+        if ((ch == '\\') && isNewLine(code.charAt(1))) {
+            // just throw away the backslash
+            code.pop();
+            return true;
+        }
+
+        return false;
+    }
+
+    private static boolean isNewLine(char ch) {
+        return (ch == '\n') || (ch == '\r');
+    }
+}

sonar-objective-c-plugin/src/main/java/org/sonar/objectivec/lexer/BackslashChannel.java

@@ -0,0 +1,114 @@
+/*
+ * SonarQube Objective-C (Community) Plugin
+ * Copyright (C) 2012-2016 OCTO Technology, Backelite, and contributors
+ * mailto:sonarqube@googlegroups.com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.objectivec.lexer;
+
+import com.sonar.sslr.api.Token;
+import com.sonar.sslr.impl.Lexer;
+import org.sonar.objectivec.api.ObjectiveCTokenType;
+import org.sonar.sslr.channel.Channel;
+import org.sonar.sslr.channel.CodeReader;
+
+/**
+ * @author Sonar C++ Plugin (Community) authors
+ */
+public class CharacterLiteralsChannel extends Channel<Lexer> {
+    private static final char EOF = (char) -1;
+
+    private final StringBuilder sb = new StringBuilder();
+
+    private int index;
+    private char ch;
+
+    @Override
+    public boolean consume(CodeReader code, Lexer output) {
+        int line = code.getLinePosition();
+        int column = code.getColumnPosition();
+        index = 0;
+        readPrefix(code);
+        if ((ch != '\'')) {
+            return false;
+        }
+        if (!read(code)) {
+            return false;
+        }
+        readUdSuffix(code);
+        for (int i = 0; i < index; i++) {
+            sb.append((char) code.pop());
+        }
+        output.addToken(Token.builder()
+                .setLine(line)
+                .setColumn(column)
+                .setURI(output.getURI())
+                .setValueAndOriginalValue(sb.toString())
+                .setType(ObjectiveCTokenType.CHARACTER_LITERAL)
+                .build());
+        sb.setLength(0);
+        return true;
+    }
+
+    private boolean read(CodeReader code) {
+        index++;
+        while (code.charAt(index) != ch) {
+            if (code.charAt(index) == EOF) {
+                return false;
+            }
+            if (code.charAt(index) == '\\') {
+                // escape
+                index++;
+            }
+            index++;
+        }
+        index++;
+        return true;
+    }
+
+    private void readPrefix(CodeReader code) {
+        ch = code.charAt(index);
+        if ((ch == 'u') || (ch == 'U') || ch == 'L') {
+            index++;
+            ch = code.charAt(index);
+        }
+    }
+
+    private void readUdSuffix(CodeReader code) {
+        for (int start_index = index, len = 0; ; index++) {
+            char c = code.charAt(index);
+            if (c == EOF) {
+                return;
+            }
+            if ((c >= 'a' && c <= 'z')
+                    || (c >= 'A' && c <= 'Z')
+                    || (c == '_')) {
+                len++;
+            } else {
+                if (c >= '0' && c <= '9') {
+                    if (len > 0) {
+                        len++;
+                    } else {
+                        index = start_index;
+                        return;
+                    }
+                } else {
+                    return;
+                }
+            }
+        }
+    }
+}