Improve tokenizer and highlighting

11 files changed, 350 insertions, 678 deletions

diff --git a/.gitignore b/.gitignore
index c4f541d..e02da25 100644
--- a/.gitignore
+++ b/.gitignore
@@ -40,3 +40,6 @@ target/
 
 # npm
 node_modules/
+
+# Visual Studio
+.vs/
diff --git a/pom.xml b/pom.xml
index e4d7bff..eed37f0 100644
--- a/pom.xml
+++ b/pom.xml
@@ -74,7 +74,7 @@
         <slf4j.version>1.7.25</slf4j.version>
         <assertj.version>3.10.0</assertj.version>
         <junit.version>4.12</junit.version>
-        <sonar.version>7.2.0.13185</sonar.version>
+        <sonar.version>7.2</sonar.version>
         <sonar.min.version>6.7</sonar.min.version>
         <sonar-orchestrator.version>3.17.0.1491</sonar-orchestrator.version>
         <sonarlint.version>3.1.0.1376</sonarlint.version>
@@ -83,6 +83,8 @@
         <commons-io.version>2.5</commons-io.version>
         <gson.version>2.6.2</gson.version>
         <sonar-analyzer-commons.version>1.7.0.243</sonar-analyzer-commons.version>
+        <sslr.version>1.22</sslr.version>
+        <commonslang.version>2.6</commonslang.version>
 
         <artifactsToPublish>${project.groupId}:sonar-css-plugin:jar</artifactsToPublish>
     </properties>
@@ -101,6 +103,16 @@
                 <version>${sonar-analyzer-commons.version}</version>
             </dependency>
             <dependency>
+                <groupId>commons-lang</groupId>
+                <artifactId>commons-lang</artifactId>
+                <version>${commonslang.version}</version>
+            </dependency>
+            <dependency>
+                <groupId>org.sonarsource.sslr</groupId>
+                <artifactId>sslr-core</artifactId>
+                <version>${sslr.version}</version>
+            </dependency>
+            <dependency>
                 <groupId>commons-io</groupId>
                 <artifactId>commons-io</artifactId>
                 <version>${commons-io.version}</version>
diff --git a/sonar-css-plugin/pom.xml b/sonar-css-plugin/pom.xml
index ee91ea8..0ad715c 100644
--- a/sonar-css-plugin/pom.xml
+++ b/sonar-css-plugin/pom.xml
@@ -59,7 +59,10 @@
         <dependency>
             <groupId>commons-lang</groupId>
             <artifactId>commons-lang</artifactId>
-            <version>2.6</version>
+        </dependency>
+        <dependency>
+            <groupId>org.sonarsource.sslr</groupId>
+            <artifactId>sslr-core</artifactId>
         </dependency>
         <dependency>
             <groupId>commons-io</groupId>
@@ -124,7 +127,7 @@
                             <rules>
                                 <requireFilesSize>
                                     <minsize>5000000</minsize>
-                                    <maxsize>6000000</maxsize>
+                                    <maxsize>6100000</maxsize>
                                     <files>
                                         <file>${project.build.directory}/${project.build.finalName}.jar</file>
                                     </files>
diff --git a/sonar-css-plugin/src/main/java/org/sonar/css/plugin/CssLexer.java b/sonar-css-plugin/src/main/java/org/sonar/css/plugin/CssLexer.java
new file mode 100644
index 0000000..9a4bb58
--- /dev/null
+++ b/sonar-css-plugin/src/main/java/org/sonar/css/plugin/CssLexer.java
@@ -0,0 +1,76 @@
+/*
+ * SonarCSS
+ * Copyright (C) 2018-2018 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.css.plugin;
+
+import com.sonar.sslr.impl.Lexer;
+
+import static com.sonar.sslr.impl.channel.RegexpChannelBuilder.regexp;
+
+// This is a at-best lexer.
+// It is far from being entirely matching the standard definition of css/less/scss tokens nor
+// following the theory of what a lexer responsibilities are but as we are only building line metrics and highlighting
+// on top of it we decided to focus on simplicity over being extensive.
+public final class CssLexer {
+
+  private static final String NEW_LINE = "(\r\n|\r|\n|\f)";
+  private static final String WHITESPACE = "[\t\n\f\r ]";
+  private static final String NON_ASCII = "[^\\p{ASCII}]";
+  private static final String HEX_DIGIT = "0-9a-fA-F";
+  private static final String ESCAPE = "(\\\\[" + HEX_DIGIT + "]{1,6}" + WHITESPACE + "?)|\\[^\r\n\f" + HEX_DIGIT + "]";
+
+  private static final String PUNCTUATOR = "[!:,;%&+#\\*-/=>\\(\\)\\[\\]\\{\\}]";
+
+  private static final String MULTI_LINE_COMMENT = "/\\*(.|" + NEW_LINE + ")*?\\*/";
+  private static final String INLINE_COMMENT = "//.*";
+  private static final String COMMENT  = "(" + INLINE_COMMENT + "|" + MULTI_LINE_COMMENT + ")";
+
+  private static final String NUMBER = "[+|-]?\\d*\\.?\\d+([a-z]+|%)?";
+
+  private static final String NAME_CHAR = "[a-zA-Z0-9_-]|" + NON_ASCII + "|" + ESCAPE;
+  private static final String NAME_START = "[a-zA-Z_]|" + NON_ASCII + "|" + ESCAPE;
+
+  private static final String IDENTIFIER = "-?(" + NAME_START + ")(" + NAME_CHAR + ")*";
+  private static final String AT_IDENTIFIER = "@+" + IDENTIFIER;
+  private static final String HASH_IDENTIFIER = "#(" + NAME_CHAR + ")+";
+  private static final String DOLLAR_IDENTIFIER = "\\$(" + NAME_CHAR + ")+";
+
+  private static final String DOUBLE_QUOTE_STRING = "~?\"([^\"\\\\\r\n\f]|" + ESCAPE + "|\\\\" + NEW_LINE + ")*\"";
+  private static final String SINGLE_QUOTE_STRING = "~?'([^'\\\\\r\n\f]|" + ESCAPE + "|\\\\" + NEW_LINE + ")*'";
+  private static final String STRING = "(" + SINGLE_QUOTE_STRING + "|" + DOUBLE_QUOTE_STRING + ")";
+
+  private CssLexer() {
+  }
+
+  public static Lexer create() {
+    return Lexer.builder()
+      .withFailIfNoChannelToConsumeOneCharacter(false)
+
+      .withChannel(regexp(CssTokenType.COMMENT, COMMENT))
+      .withChannel(regexp(CssTokenType.STRING, STRING))
+      .withChannel(regexp(CssTokenType.AT_IDENTIFIER, AT_IDENTIFIER))
+      .withChannel(regexp(CssTokenType.HASH_IDENTIFIER, HASH_IDENTIFIER))
+      .withChannel(regexp(CssTokenType.DOLLAR_IDENTIFIER, DOLLAR_IDENTIFIER))
+      .withChannel(regexp(CssTokenType.IDENTIFIER, IDENTIFIER))
+      .withChannel(regexp(CssTokenType.NUMBER, NUMBER))
+      .withChannel(regexp(CssTokenType.PUNCTUATOR, PUNCTUATOR))
+
+      .build();
+  }
+}
diff --git a/sonar-css-plugin/src/main/java/org/sonar/css/plugin/CssToken.java b/sonar-css-plugin/src/main/java/org/sonar/css/plugin/CssToken.java
new file mode 100644
index 0000000..4ba0cc6
--- /dev/null
+++ b/sonar-css-plugin/src/main/java/org/sonar/css/plugin/CssToken.java
@@ -0,0 +1,45 @@
+/*
+ * SonarCSS
+ * Copyright (C) 2018-2018 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.css.plugin;
+
+import com.sonar.sslr.api.Token;
+import com.sonar.sslr.api.TokenType;
+import org.sonarsource.analyzer.commons.TokenLocation;
+
+public class CssToken {
+  CssTokenType type;
+  String text;
+  Integer startLine;
+  Integer startColumn;
+  Integer endLine;
+  Integer endColumn;
+
+  public CssToken(Token token) {
+    TokenType tokenType = token.getType();
+    this.type = (CssTokenType)tokenType;
+    this.text = token.getValue();
+
+    TokenLocation tokenLocation = new TokenLocation(token.getLine(), token.getColumn(), token.getValue());
+    this.startLine = tokenLocation.startLine();
+    this.startColumn = tokenLocation.startLineOffset();
+    this.endLine = tokenLocation.endLine();
+    this.endColumn = tokenLocation.endLineOffset();
+  }
+}
diff --git a/sonar-css-plugin/src/main/java/org/sonar/css/plugin/Token.java b/sonar-css-plugin/src/main/java/org/sonar/css/plugin/CssTokenType.java
index dc9af61..dccc5b7 100644
--- a/sonar-css-plugin/src/main/java/org/sonar/css/plugin/Token.java
+++ b/sonar-css-plugin/src/main/java/org/sonar/css/plugin/CssTokenType.java
@@ -19,30 +19,31 @@
  */
 package org.sonar.css.plugin;
 
-public class Token {
+import com.sonar.sslr.api.AstNode;
+import com.sonar.sslr.api.TokenType;
 
-  public enum Type {
-    COMMENT,
-    STRING,
-    WORD,
-    AT_WORD,
-    BRACKETS,
-    PUNCTUATOR
+public enum CssTokenType implements TokenType {
+  COMMENT,
+  PUNCTUATOR,
+  NUMBER,
+  STRING,
+  AT_IDENTIFIER,
+  HASH_IDENTIFIER,
+  DOLLAR_IDENTIFIER,
+  IDENTIFIER;
+
+  @Override
+  public String getName() {
+    return name();
   }
 
-  Type type;
-  String text;
-  Integer startLine;
-  Integer startColumn;
-  Integer endLine;
-  Integer endColumn;
+  @Override
+  public String getValue() {
+    return name();
+  }
 
-  public Token(Type type, String text, Integer startLine, Integer startColumn, Integer endLine, Integer endColumn) {
-    this.text = text;
-    this.type = type;
-    this.startLine = startLine;
-    this.startColumn = startColumn;
-    this.endLine = endLine;
-    this.endColumn = endColumn;
+  @Override
+  public boolean hasToBeSkippedFromAst(AstNode node) {
+    return false;
   }
 }
diff --git a/sonar-css-plugin/src/main/java/org/sonar/css/plugin/MetricSensor.java b/sonar-css-plugin/src/main/java/org/sonar/css/plugin/MetricSensor.java
index abbbd50..6257b74 100644
--- a/sonar-css-plugin/src/main/java/org/sonar/css/plugin/MetricSensor.java
+++ b/sonar-css-plugin/src/main/java/org/sonar/css/plugin/MetricSensor.java
@@ -21,7 +21,6 @@ package org.sonar.css.plugin;
 
 import java.io.IOException;
 import java.util.List;
-import javax.script.ScriptException;
 import org.sonar.api.batch.fs.FileSystem;
 import org.sonar.api.batch.fs.InputFile;
 import org.sonar.api.batch.sensor.Sensor;
@@ -57,11 +56,11 @@ public class MetricSensor implements Sensor {
   private static void saveHighlights(SensorContext sensorContext, InputFile input, Tokenizer tokenizer) {
     try {
       NewHighlighting highlighting = sensorContext.newHighlighting().onFile(input);
-      List<Token> tokenList = tokenizer.tokenize(input.contents());
+      List<CssToken> tokenList = tokenizer.tokenize(input.contents());
 
       for (int i = 0; i < tokenList.size(); i++) {
-        Token currentToken = tokenList.get(i);
-        Token nextToken = i + 1 == tokenList.size() ? null : tokenList.get(i + 1);
+        CssToken currentToken = tokenList.get(i);
+        CssToken nextToken = i + 1 < tokenList.size() ? tokenList.get(i + 1) : null;
 
         TypeOfText highlightingType = null;
         switch (currentToken.type) {
@@ -73,18 +72,32 @@ public class MetricSensor implements Sensor {
             highlightingType = TypeOfText.STRING;
             break;
 
-          case WORD:
-            if (Character.isDigit(currentToken.text.charAt(0)) || currentToken.text.matches("^#[0-9a-fA-F]+$")) {
+          case NUMBER:
+            highlightingType = TypeOfText.CONSTANT;
+            break;
+
+          case AT_IDENTIFIER:
+            highlightingType = TypeOfText.ANNOTATION;
+            break;
+
+          case DOLLAR_IDENTIFIER:
+            highlightingType = TypeOfText.KEYWORD;
+            break;
+
+          case HASH_IDENTIFIER:
+            if (currentToken.text.matches("^#[0-9a-fA-F]+$")) {
               highlightingType = TypeOfText.CONSTANT;
-            } else if (nextToken != null && nextToken.text.equals(":")) {
-              highlightingType = TypeOfText.KEYWORD_LIGHT;
-            } else if (currentToken.text.startsWith(".") || (nextToken != null && nextToken.text.startsWith("{"))) {
+            } else {
               highlightingType = TypeOfText.KEYWORD;
             }
             break;
 
-          case AT_WORD:
-            highlightingType = TypeOfText.ANNOTATION;
+          case IDENTIFIER:
+            // We want to highlight the property key of a css/scss/less file and as the tokenizer is putting the ':' into another token
+            // we need to look for identifier followed by a PUNCTUATOR token with text ':'.
+            if (nextToken != null && nextToken.text.equals(":")) {
+              highlightingType = TypeOfText.KEYWORD_LIGHT;
+            }
             break;
 
           default:
@@ -92,14 +105,12 @@ public class MetricSensor implements Sensor {
         }
 
         if (highlightingType != null) {
-          highlighting.highlight(currentToken.startLine, currentToken.startColumn - 1, currentToken.endLine, currentToken.endColumn, highlightingType);
+          highlighting.highlight(currentToken.startLine, currentToken.startColumn, currentToken.endLine, currentToken.endColumn, highlightingType);
         }
       }
 
       highlighting.save();
 
-    } catch (ScriptException e) {
-      LOG.error(String.format("Failed to tokenize file '%s'", input.toString()), e);
     } catch (IOException e) {
       LOG.error(String.format("Failed to read file '%s'", input.toString()), e);
     }
diff --git a/sonar-css-plugin/src/main/java/org/sonar/css/plugin/Tokenizer.java b/sonar-css-plugin/src/main/java/org/sonar/css/plugin/Tokenizer.java
index 220bfaa..cf84e08 100644
--- a/sonar-css-plugin/src/main/java/org/sonar/css/plugin/Tokenizer.java
+++ b/sonar-css-plugin/src/main/java/org/sonar/css/plugin/Tokenizer.java
@@ -1,133 +1,38 @@
-/*
- * SonarCSS
- * Copyright (C) 2018-2018 SonarSource SA
- * mailto:info AT sonarsource DOT com
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 3 of the License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public License
- * along with this program; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
- */
-package org.sonar.css.plugin;
-
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.nio.charset.StandardCharsets;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-import javax.script.ScriptEngine;
-import javax.script.ScriptException;
-import jdk.nashorn.api.scripting.NashornScriptEngineFactory;
-import org.apache.commons.lang.StringEscapeUtils;
-import org.sonar.css.plugin.Token.Type;
-
-public class Tokenizer {
-
-  public List<Token> tokenize(String css) throws ScriptException {
-    ScriptEngine engine = new NashornScriptEngineFactory().getScriptEngine();
-    InputStream tokenizeScript = Tokenizer.class.getClassLoader().getResourceAsStream("tokenize.js");
-    engine.eval(new InputStreamReader(tokenizeScript, StandardCharsets.UTF_8));
-    String cssInput = "tokenize('" + StringEscapeUtils.escapeJavaScript(css) + "')";
-    Object tokens = engine.eval(cssInput);
-    return extractTokens(tokens);
-  }
-
-  private static List<Token> extractTokens(Object tokens) {
-    // tokens is result of call to javascript function tokenize(). It returns an array of arrays, where nested arrays
-    // correspond to tokens. These array javascript objects mapped in Java to Map objects where array index is key.
-
-    List<Token> resultList = new ArrayList<>();
-    for (Object tokenObject : ((Map<String, Object>) tokens).values()) {
-
-      // Access the inner arrays (disregard the keys) and use their length to decide which type of token we are
-      // dealing with.
-      Map<String, Object> tokenProperties = (Map<String, Object>) tokenObject;
-
-      // skip whitespace token (size < 4)
-      if (tokenProperties.size() >= 4) {
-        String text = tokenProperties.get("1").toString();
-        Type type = computeType(tokenProperties.get("0").toString(), text);
-        Integer startLine = convertToInt(tokenProperties.get("2"));
-        Integer startColumn = ((Double) tokenProperties.get("3")).intValue();
-
-        // all cases except for punctuator type
-        if (tokenProperties.size() == 6) {
-          Integer endLine = convertToInt(tokenProperties.get("4"));
-          Integer endColumn = ((Double) tokenProperties.get("5")).intValue();
-
-
-          if (isTokenWithPunctuator(text, ",", startLine, endLine)) {
-            resultList.addAll(splitTokenWithPunctuator(text, type, startLine, startColumn, endLine, endColumn));
-          } else if (isTokenWithPunctuator(text, ":", startLine, endLine)) {
-            resultList.addAll(splitTokenWithPunctuator(text, type, startLine, startColumn, endLine, endColumn));
-          } else {
-            resultList.add(new Token(type, text, startLine, startColumn, endLine, endColumn));
-          }
-        } else {
-          // is punctuator
-          resultList.add(new Token(type, text, startLine, startColumn, startLine, startColumn));
-        }
-      }
-    }
-
-    return resultList;
-  }
-
-  // Javascript tokenizer is not returning 2 tokens for words ending with a comma (e.g. foo,) and for words starting
-  // with at symbol and endings with colon (e.g. @base:) so we need to split the word into 2 tokens (1 word without
-  // the punctuator and 1 punctuator).
-  // For the sake of simplicity we don't handle words ending with the punctuator on a new line.
-  private static Boolean isTokenWithPunctuator(String text, String punctuator, Integer startLine, Integer endLine) {
-    return text.length() > 1 && text.endsWith(punctuator) && startLine.equals(endLine);
-  }
-
-  private static List<Token> splitTokenWithPunctuator(String text, Type type, Integer startLine, Integer startColumn, Integer endLine, Integer endColumn) {
-    List<Token> tokenList = new ArrayList<>();
-
-    tokenList.add(new Token(type, text.substring(0, text.length() - 1), startLine, startColumn, endLine, endColumn - 1));
-    tokenList.add(new Token(Type.PUNCTUATOR, text.substring(text.length() - 1), startLine, endColumn, endLine, endColumn));
-
-    return tokenList;
-  }
-
-  private static Integer convertToInt(Object value) {
-    if (value instanceof Double) {
-      return ((Double) value).intValue();
-    } else if (value instanceof Integer) {
-      return (Integer) value;
-    } else {
-      throw new IllegalStateException("Failed to convert to number: " + value);
-    }
-  }
-
-  private static Type computeType(String type, String text) {
-    switch (type) {
-      case "at-word":
-        return Type.AT_WORD;
-      case "word":
-        if (",".equals(text)) {
-          return Type.PUNCTUATOR;
-        } else {
-          return Type.WORD;
-        }
-      case "comment":
-        return Type.COMMENT;
-      case "string":
-        return Type.STRING;
-      case "brackets":
-        return Type.BRACKETS;
-      default:
-        return Type.PUNCTUATOR;
-    }
-  }
-}
+/*

+ * SonarCSS

+ * Copyright (C) 2018-2018 SonarSource SA

+ * mailto:info AT sonarsource DOT com

+ *

+ * This program is free software; you can redistribute it and/or

+ * modify it under the terms of the GNU Lesser General Public

+ * License as published by the Free Software Foundation; either

+ * version 3 of the License, or (at your option) any later version.

+ *

+ * This program is distributed in the hope that it will be useful,

+ * but WITHOUT ANY WARRANTY; without even the implied warranty of

+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU

+ * Lesser General Public License for more details.

+ *

+ * You should have received a copy of the GNU Lesser General Public License

+ * along with this program; if not, write to the Free Software Foundation,

+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.

+ */

+package org.sonar.css.plugin;

+

+import com.sonar.sslr.api.Token;

+import java.util.ArrayList;

+import java.util.List;

+import java.util.stream.Collectors;

+

+public class Tokenizer {

+

+  public List<CssToken> tokenize(String css) {

+    List<Token> tokenList = CssLexer.create().lex(css);

+

+    // remove last token (EOF token)

+    List<Token> cloneTokenList = new ArrayList<>(tokenList);

+    cloneTokenList.remove(cloneTokenList.size() - 1);

+

+    return cloneTokenList.stream().map(CssToken::new).collect(Collectors.toList());

+  }

+}

diff --git a/sonar-css-plugin/src/main/resources/tokenize.js b/sonar-css-plugin/src/main/resources/tokenize.js
deleted file mode 100644
index bbf9b1b..0000000
--- a/sonar-css-plugin/src/main/resources/tokenize.js
+++ /dev/null
@@ -1,374 +0,0 @@
-/*
- * SonarCSS
- * Copyright (C) 2018-2018 SonarSource SA
- * mailto:info AT sonarsource DOT com
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 3 of the License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public License
- * along with this program; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
- */
-
-/*
- * The MIT License (MIT)
- *
- * Copyright 2013 Andrey Sitnik <andrey@sitnik.ru>
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy of
- * this software and associated documentation files (the "Software"), to deal in
- * the Software without restriction, including without limitation the rights to
- * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
- * the Software, and to permit persons to whom the Software is furnished to do so,
- * subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in all
- * copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
- * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
- * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
- * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
- */
-
-var SINGLE_QUOTE      = '\''.charCodeAt(0);
-var DOUBLE_QUOTE      =  '"'.charCodeAt(0);
-var BACKSLASH         = '\\'.charCodeAt(0);
-var SLASH             =  '/'.charCodeAt(0);
-var NEWLINE           = '\n'.charCodeAt(0);
-var SPACE             =  ' '.charCodeAt(0);
-var FEED              = '\f'.charCodeAt(0);
-var TAB               = '\t'.charCodeAt(0);
-var CR                = '\r'.charCodeAt(0);
-var OPEN_SQUARE       =  '['.charCodeAt(0);
-var CLOSE_SQUARE      =  ']'.charCodeAt(0);
-var OPEN_PARENTHESES  =  '('.charCodeAt(0);
-var CLOSE_PARENTHESES =  ')'.charCodeAt(0);
-var OPEN_CURLY        =  '{'.charCodeAt(0);
-var CLOSE_CURLY       =  '}'.charCodeAt(0);
-var SEMICOLON         =  ';'.charCodeAt(0);
-var ASTERISK          =  '*'.charCodeAt(0);
-var COLON             =  ':'.charCodeAt(0);
-var AT                =  '@'.charCodeAt(0);
-
-var RE_AT_END      = /[ \n\t\r\f\{\(\)'"\\;/\[\]#]/g;
-var RE_WORD_END    = /[ \n\t\r\f\(\)\{\}:;@!'"\\\]\[#]|\/(?=\*)/g;
-var RE_BAD_BRACKET = /.[\\\/\("'\n]/;
-var RE_HEX_ESCAPE  = /[a-f0-9]/i;
-
-function tokenize(css) {
-	var input = {css: css, error: function() {}};
-	var processor = tokenizer(input);
-    var tokens = [];
-    while (!processor.endOfFile()) {
-        tokens.push(processor.nextToken());
-    }
-    return tokens;
-}
-
-function tokenizer(input ) {
-	var options = {};
-    var css = input.css.valueOf();
-    var ignore = options.ignoreErrors;
-
-    var code, next, quote, lines, last, content, escape,
-        nextLine, nextOffset, escaped, escapePos, prev, n, currentToken;
-
-    var length = css.length;
-    var offset = -1;
-    var line = 1;
-    var pos = 0;
-    var buffer = [];
-    var returned = [];
-
-    function unclosed(what) {
-        throw input.error('Unclosed ' + what, line, pos - offset);
-    }
-
-    function endOfFile() {
-        return returned.length === 0 && pos >= length;
-    }
-
-    function nextToken() {
-        if ( returned.length ) return returned.pop();
-        if ( pos >= length ) return;
-
-        code = css.charCodeAt(pos);
-        if ( code === NEWLINE || code === FEED ||
-             code === CR && css.charCodeAt(pos + 1) !== NEWLINE ) {
-            offset = pos;
-            line += 1;
-        }
-
-        switch ( code ) {
-        case NEWLINE:
-        case SPACE:
-        case TAB:
-        case CR:
-        case FEED:
-            next = pos;
-            do {
-                next += 1;
-                code = css.charCodeAt(next);
-                if ( code === NEWLINE ) {
-                    offset = next;
-                    line += 1;
-                }
-            } while ( code === SPACE   ||
-                      code === NEWLINE ||
-                      code === TAB     ||
-                      code === CR      ||
-                      code === FEED );
-
-            currentToken = ['space', css.slice(pos, next)];
-            pos = next - 1;
-            break;
-
-        case OPEN_SQUARE:
-            currentToken = ['[', '[', line, pos - offset];
-            break;
-
-        case CLOSE_SQUARE:
-            currentToken = [']', ']', line, pos - offset];
-            break;
-
-        case OPEN_CURLY:
-            currentToken = ['{', '{', line, pos - offset];
-            break;
-
-        case CLOSE_CURLY:
-            currentToken = ['}', '}', line, pos - offset];
-            break;
-
-        case COLON:
-            currentToken = [':', ':', line, pos - offset];
-            break;
-
-        case SEMICOLON:
-            currentToken = [';', ';', line, pos - offset];
-            break;
-
-        case OPEN_PARENTHESES:
-            prev = buffer.length ? buffer.pop()[1] : '';
-            n    = css.charCodeAt(pos + 1);
-            if ( prev === 'url' &&
-                 n !== SINGLE_QUOTE && n !== DOUBLE_QUOTE &&
-                 n !== SPACE && n !== NEWLINE && n !== TAB &&
-                 n !== FEED && n !== CR ) {
-                next = pos;
-                do {
-                    escaped = false;
-                    next    = css.indexOf(')', next + 1);
-                    if ( next === -1 ) {
-                        if ( ignore ) {
-                            next = pos;
-                            break;
-                        } else {
-                            unclosed('bracket');
-                        }
-                    }
-                    escapePos = next;
-                    while ( css.charCodeAt(escapePos - 1) === BACKSLASH ) {
-                        escapePos -= 1;
-                        escaped = !escaped;
-                    }
-                } while ( escaped );
-
-                currentToken = ['brackets', css.slice(pos, next + 1),
-                    line, pos  - offset,
-                    line, next - offset
-                ];
-
-                pos = next;
-
-            } else {
-                next    = css.indexOf(')', pos + 1);
-                content = css.slice(pos, next + 1);
-
-                if ( next === -1 || RE_BAD_BRACKET.test(content) ) {
-                    currentToken = ['(', '(', line, pos - offset];
-                } else {
-                    currentToken = ['brackets', content,
-                        line, pos  - offset,
-                        line, next - offset
-                    ];
-                    pos = next;
-                }
-            }
-
-            break;
-
-        case CLOSE_PARENTHESES:
-            currentToken = [')', ')', line, pos - offset];
-            break;
-
-        case SINGLE_QUOTE:
-        case DOUBLE_QUOTE:
-            quote = code === SINGLE_QUOTE ? '\'' : '"';
-            next  = pos;
-            do {
-                escaped = false;
-                next    = css.indexOf(quote, next + 1);
-                if ( next === -1 ) {
-                    if ( ignore ) {
-                        next = pos + 1;
-                        break;
-                    } else {
-                        unclosed('string');
-                    }
-                }
-                escapePos = next;
-                while ( css.charCodeAt(escapePos - 1) === BACKSLASH ) {
-                    escapePos -= 1;
-                    escaped = !escaped;
-                }
-            } while ( escaped );
-
-            content = css.slice(pos, next + 1);
-            lines   = content.split('\n');
-            last    = lines.length - 1;
-
-            if ( last > 0 ) {
-                nextLine   = line + last;
-                nextOffset = next - lines[last].length;
-            } else {
-                nextLine   = line;
-                nextOffset = offset;
-            }
-
-            currentToken = ['string', css.slice(pos, next + 1),
-                line, pos  - offset,
-                nextLine, next - nextOffset
-            ];
-
-            offset = nextOffset;
-            line   = nextLine;
-            pos    = next;
-            break;
-
-        case AT:
-            RE_AT_END.lastIndex = pos + 1;
-            RE_AT_END.test(css);
-            if ( RE_AT_END.lastIndex === 0 ) {
-                next = css.length - 1;
-            } else {
-                next = RE_AT_END.lastIndex - 2;
-            }
-
-            currentToken = ['at-word', css.slice(pos, next + 1),
-                line, pos  - offset,
-                line, next - offset
-            ];
-
-            pos = next;
-            break;
-
-        case BACKSLASH:
-            next   = pos;
-            escape = true;
-            while ( css.charCodeAt(next + 1) === BACKSLASH ) {
-                next  += 1;
-                escape = !escape;
-            }
-            code = css.charCodeAt(next + 1);
-            if ( escape && (code !== SLASH   &&
-                            code !== SPACE   &&
-                            code !== NEWLINE &&
-                            code !== TAB     &&
-                            code !== CR      &&
-                            code !== FEED ) ) {
-                next += 1;
-                if ( RE_HEX_ESCAPE.test(css.charAt(next)) ) {
-                    while ( RE_HEX_ESCAPE.test(css.charAt(next + 1)) ) {
-                        next += 1;
-                    }
-                    if ( css.charCodeAt(next + 1) === SPACE ) {
-                        next += 1;
-                    }
-                }
-            }
-
-            currentToken = ['word', css.slice(pos, next + 1),
-                line, pos  - offset,
-                line, next - offset
-            ];
-
-            pos = next;
-            break;
-
-        default:
-            if ( code === SLASH && css.charCodeAt(pos + 1) === ASTERISK ) {
-                next = css.indexOf('*/', pos + 2) + 1;
-                if ( next === 0 ) {
-                    if ( ignore ) {
-                        next = css.length;
-                    } else {
-                        unclosed('comment');
-                    }
-                }
-
-                content = css.slice(pos, next + 1);
-                lines   = content.split('\n');
-                last    = lines.length - 1;
-
-                if ( last > 0 ) {
-                    nextLine   = line + last;
-                    nextOffset = next - lines[last].length;
-                } else {
-                    nextLine   = line;
-                    nextOffset = offset;
-                }
-
-                currentToken = ['comment', content,
-                    line,     pos  - offset,
-                    nextLine, next - nextOffset
-                ];
-
-                offset = nextOffset;
-                line   = nextLine;
-                pos    = next;
-
-            } else {
-                RE_WORD_END.lastIndex = pos + 1;
-                RE_WORD_END.test(css);
-                if ( RE_WORD_END.lastIndex === 0 ) {
-                    next = css.length - 1;
-                } else {
-                    next = RE_WORD_END.lastIndex - 2;
-                }
-
-                currentToken = ['word', css.slice(pos, next + 1),
-                    line, pos  - offset,
-                    line, next - offset
-                ];
-
-                buffer.push(currentToken);
-
-                pos = next;
-            }
-
-            break;
-        }
-
-        pos++;
-        return currentToken;
-    }
-
-    function back(token) {
-        returned.push(token);
-    }
-
-    return {
-        back:back, nextToken:nextToken, endOfFile:endOfFile
-    };
-}
diff --git a/sonar-css-plugin/src/test/java/org/sonar/css/plugin/MetricSensorTest.java b/sonar-css-plugin/src/test/java/org/sonar/css/plugin/MetricSensorTest.java
index 6af504b..e8eb31b 100644
--- a/sonar-css-plugin/src/test/java/org/sonar/css/plugin/MetricSensorTest.java
+++ b/sonar-css-plugin/src/test/java/org/sonar/css/plugin/MetricSensorTest.java
@@ -71,7 +71,7 @@ public class MetricSensorTest {
     highlight("\"foo\"");
     assertHighlighting(1, 0, 5, TypeOfText.STRING);
 
-    highlight("\"foo\nbar\"");
+    highlight("\"foo\\\nbar\"");
     assertHighlighting(1, 0, 4, TypeOfText.STRING);
     assertHighlighting(2, 0, 4, TypeOfText.STRING);
   }
@@ -111,22 +111,11 @@ public class MetricSensorTest {
 
   @Test
   public void keyword() throws IOException {
-    highlight("foo { }");
-    assertHighlighting(1, 0, 3, TypeOfText.KEYWORD);
-
-    highlight(".foo { }");
-    assertHighlighting(1, 0, 4, TypeOfText.KEYWORD);
-
-    highlight(".foo bar { }");
+    highlight("$foo { }");
     assertHighlighting(1, 0, 4, TypeOfText.KEYWORD);
-    assertHighlighting(1, 5, 3, TypeOfText.KEYWORD);
-
-    highlight(".border-radius(@radius) { }");
-    assertHighlighting(1, 0, 14, TypeOfText.KEYWORD);
 
     highlight("#header { .border-radius(4px); }");
     assertHighlighting(1, 0, 7, TypeOfText.KEYWORD);
-    assertHighlighting(1, 10, 14, TypeOfText.KEYWORD);
   }
 
   @Test
diff --git a/sonar-css-plugin/src/test/java/org/sonar/css/plugin/TokenizerTest.java b/sonar-css-plugin/src/test/java/org/sonar/css/plugin/TokenizerTest.java
index b1589c0..76c4617 100644
--- a/sonar-css-plugin/src/test/java/org/sonar/css/plugin/TokenizerTest.java
+++ b/sonar-css-plugin/src/test/java/org/sonar/css/plugin/TokenizerTest.java
@@ -20,11 +20,7 @@
 package org.sonar.css.plugin;
 
 import java.util.List;
-import java.util.Optional;
-
-import javax.script.ScriptException;
 import org.junit.Test;
-import org.sonar.css.plugin.Token.Type;
 
 import static org.assertj.core.api.Assertions.assertThat;
 
@@ -33,67 +29,84 @@ public class TokenizerTest {
   private final static Tokenizer tokenizer = new Tokenizer();
   
   @Test
-  public void word() throws ScriptException {
-    assertToken("bar { }", 0, "bar", Type.WORD);
-    assertToken("bar: foo { }", 0, "bar", Type.WORD);
-    assertToken("bar: foo-baz { }", 2, "foo-baz", Type.WORD);
-    assertToken("foo bar { }", 1, "bar", Type.WORD);
-    assertToken("#bar { }", 0, "#bar", Type.WORD);
-    assertToken("foo.bar { }", 0, "foo.bar", Type.WORD);
-    assertToken(".bar { }", 0, ".bar", Type.WORD);
-    assertToken("bar { foo: 42; }", 2, "foo", Type.WORD);
-    assertToken("bar { foo: baz; }", 4, "baz", Type.WORD);
-    assertToken("foo , bar { }", 2, "bar", Type.WORD);
+  public void identifier() {
+    assertToken("bar { }", 0, "bar", CssTokenType.IDENTIFIER);
+    assertToken("bar: foo { }", 0, "bar", CssTokenType.IDENTIFIER);
+    assertToken("bar: foo-baz { }", 2, "foo-baz", CssTokenType.IDENTIFIER);
+    assertToken("foo bar { }", 1, "bar", CssTokenType.IDENTIFIER);
+    assertToken("foo.bar { }", 0, "foo", CssTokenType.IDENTIFIER);
+    assertToken(".bar { }", 1, "bar", CssTokenType.IDENTIFIER);
+    assertToken("bar { foo: 42; }", 2, "foo", CssTokenType.IDENTIFIER);
+    assertToken("bar { foo: baz; }", 4, "baz", CssTokenType.IDENTIFIER);
+    assertToken("foo , bar { }", 2, "bar", CssTokenType.IDENTIFIER);
+
+    // support unicode characters
+    assertToken("\u03A9 { }", 0, "\u03A9", CssTokenType.IDENTIFIER);
+  }
+
+  @Test
+  public void at_identifier() {
+    assertToken("@bar { }", 0, "@bar", CssTokenType.AT_IDENTIFIER);
+  }
+
+  @Test
+  public void hash_identifier() {
+    assertToken("#bar { }", 0, "#bar", CssTokenType.HASH_IDENTIFIER);
+    assertToken("bar { color: #333; }", 4, "#333", CssTokenType.HASH_IDENTIFIER);
+    assertToken("bar { color: #e535ab; }", 4, "#e535ab", CssTokenType.HASH_IDENTIFIER);
   }
 
   @Test
-  public void semi_colon() throws ScriptException {
-    assertToken("bar: foo { }", 1, ":", Type.PUNCTUATOR);
-    assertToken("bar { foo; }", 3, ";", Type.PUNCTUATOR);
+  public void semi_colon() {
+    assertToken("bar { foo; }", 3, ";", CssTokenType.PUNCTUATOR);
   }
 
   @Test
-  public void comma() throws ScriptException {
-    assertToken("foo , bar { }", 1, ",", Type.PUNCTUATOR);
-    assertToken("foo, bar { }", 1, ",", Type.PUNCTUATOR);
+  public void colon() {
+    assertToken("bar { foo: 2px; }", 3, ":", CssTokenType.PUNCTUATOR);
   }
 
   @Test
-  public void number_as_word() throws ScriptException {
-    assertToken("bar { foo: 1.15; }", 4, "1.15", Type.WORD);
-    assertToken("bar { foo: 1; }", 4, "1", Type.WORD);
-    assertToken("bar { foo: 1.15px; }", 4, "1.15px", Type.WORD);
-    assertToken("bar { foo: 1.15%; }", 4, "1.15%", Type.WORD);
-    assertToken("bar { foo: 1px; }", 4, "1px", Type.WORD);
-    assertToken("bar { foo: 1em/150%; }", 4, "1em/150%", Type.WORD);
+  public void comma() {
+    assertToken("foo , bar { }", 1, ",", CssTokenType.PUNCTUATOR);
+    assertToken("foo, bar { }", 1, ",", CssTokenType.PUNCTUATOR);
   }
 
   @Test
-  public void brackets() throws ScriptException {
-    assertToken("bar { foo: (1.15); }", 4, "(1.15)", Type.BRACKETS);
-    assertToken("bar { foo: ( 1.15 ); }", 4, "( 1.15 )", Type.BRACKETS);
-    assertToken("bar { foo: (1.15 1 0px); }", 4, "(1.15 1 0px)", Type.BRACKETS);
-    assertToken("bar { foo: (1.15, 1, 0px); }", 4, "(1.15, 1, 0px)", Type.BRACKETS);
-    assertToken("bar { content: string(doctitle); }", 5, "(doctitle)", Type.BRACKETS);
-    assertToken("bar { string-set: booktitle content(); }", 6, "()", Type.BRACKETS);
-    assertToken("bar { a: b(attr(href, url), c) \")\"; }", 7, "(href, url)", Type.BRACKETS);
+  public void number() {
+    assertToken("1.15", 0, "1.15", CssTokenType.NUMBER);
+    assertToken("1", 0, "1", CssTokenType.NUMBER);
+    assertToken("1.15px", 0, "1.15px", CssTokenType.NUMBER);
+    assertToken("1.15%", 0, "1.15%", CssTokenType.NUMBER);
+    assertToken("1px", 0, "1px", CssTokenType.NUMBER);
+    assertToken("1em/150%", 0, "1em", CssTokenType.NUMBER);
   }
 
   @Test
-  public void strings() throws ScriptException {
-    assertToken("bar { foo: \"\"; }", 4, "\"\"", Type.STRING);
-    assertToken("bar { foo: \"hello, world\"; }", 4, "\"hello, world\"", Type.STRING);
+  public void parenthesis() {
+    assertToken("bar { foo: (1.15); }", 4, "(", CssTokenType.PUNCTUATOR);
+    assertToken("bar { foo: (1.15); }", 6, ")", CssTokenType.PUNCTUATOR);
+    assertToken("bar { foo: ( 1.15 ); }", 4, "(", CssTokenType.PUNCTUATOR);
+    assertToken("bar { foo: (1.15 1 0px); }", 4, "(", CssTokenType.PUNCTUATOR);
+    assertToken("bar { foo: (1.15, 1, 0px); }", 4, "(", CssTokenType.PUNCTUATOR);
+    assertToken("bar { content: string(doctitle); }", 5, "(", CssTokenType.PUNCTUATOR);
+    assertToken("bar { string-set: booktitle content(); }", 6, "(", CssTokenType.PUNCTUATOR);
+    assertToken("bar { a: b(attr(href, url), c) \")\"; }", 7, "(", CssTokenType.PUNCTUATOR);
   }
 
   @Test
-  public void at_word() throws ScriptException {
-    assertToken("@bar { }", 0, "@bar", Type.AT_WORD);
+  public void strings() {
+    assertToken("bar { foo: \"text\"; }", 4, "\"text\"", CssTokenType.STRING);
+    assertToken("bar { foo: \"hello, world\"; }", 4, "\"hello, world\"", CssTokenType.STRING);
+    assertToken("bar { foo: \"\"; }", 4, "\"\"", CssTokenType.STRING);
+    assertToken("\"foo\\\nbar\"", 0, "\"foo\\\nbar\"", CssTokenType.STRING);
+    assertToken("@min768: ~\"(min-width: 768px)\"", 2, "~\"(min-width: 768px)\"", CssTokenType.STRING);
   }
 
   @Test
-  public void comment() throws ScriptException {
-    assertToken("/* foo */", 0, "/* foo */", Type.COMMENT);
-    assertToken("foo { a: /* foo */ 42; }", 4, "/* foo */", Type.COMMENT);
+  public void comment() {
+    assertToken("/* foo */", 0, "/* foo */", CssTokenType.COMMENT);
+    assertToken("foo { a: /* foo */ 42; }", 4, "/* foo */", CssTokenType.COMMENT);
     assertToken("/* \n"
       + "  this is a comment\n"
       + "  and it is awesome because\n"
@@ -102,122 +115,110 @@ public class TokenizerTest {
       + "  this is a comment\n"
       + "  and it is awesome because\n"
       + "  it is multiline!\n"
-      + "*/", Type.COMMENT, 1, 1, 5, 2);
-    assertToken("foo { a: /* foo\nbar*/ 42; }", 4, "/* foo\nbar*/", Type.COMMENT, 1, 10, 2, 5);
+      + "*/", CssTokenType.COMMENT, 1, 0, 5, 2);
+    assertToken("foo { a: /* foo\nbar*/ 42; }", 4, "/* foo\nbar*/", CssTokenType.COMMENT, 1, 9, 2, 5);
   }
 
   @Test
-  public void hashtag() throws ScriptException {
-    assertToken("bar { color: #333; }", 4, "#333", Type.WORD);
-    assertToken("bar { color: #e535ab; }", 4, "#e535ab", Type.WORD);
-    assertToken("#table-of-contents + ul li { list-style: none; }", 0, "#table-of-contents", Type.WORD);
-  }
-
-  @Test
-  public void scss_variable() throws ScriptException {
-    assertToken("$font-stack: Helvetica;", 0, "$font-stack", Type.WORD);
-    assertToken("$message-color: blue !default;", 3, "!default", Type.WORD);
-
-    List<Token> tokenList = tokenizer.tokenize("p.message-#{$alertClass} { color: red; }");
-    assertThat(tokenList.size()).isEqualTo(11);
-    assertToken(tokenList, 0, "p.message-", Type.WORD);
-    assertToken(tokenList, 1, "#", Type.WORD);
-    assertToken(tokenList, 2, "{", Type.PUNCTUATOR);
-    assertToken(tokenList, 3, "$alertClass", Type.WORD);
-    assertToken(tokenList, 4, "}", Type.PUNCTUATOR);
-    assertToken(tokenList, 5, "{", Type.PUNCTUATOR);
-    assertToken(tokenList, 6, "color", Type.WORD);
-    assertToken(tokenList, 7, ":", Type.PUNCTUATOR);
-    assertToken(tokenList, 8, "red", Type.WORD);
-    assertToken(tokenList, 9, ";", Type.PUNCTUATOR);
-    assertToken(tokenList, 10, "}", Type.PUNCTUATOR);
+  public void scss_variable() {
+    assertToken("$font-stack: Helvetica;", 0, "$font-stack", CssTokenType.DOLLAR_IDENTIFIER);
+    assertToken("$message-color: blue !default;", 4, "default", CssTokenType.IDENTIFIER);
 
+    List<CssToken> tokenList = tokenizer.tokenize("p.message-#{$alertClass} { color: red; }");
+    assertThat(tokenList.size()).isEqualTo(13);
+    assertToken(tokenList, 0, "p", CssTokenType.IDENTIFIER);
+    assertToken(tokenList, 1, ".", CssTokenType.PUNCTUATOR);
+    assertToken(tokenList, 2, "message-", CssTokenType.IDENTIFIER);
+    assertToken(tokenList, 3, "#", CssTokenType.PUNCTUATOR);
+    assertToken(tokenList, 4, "{", CssTokenType.PUNCTUATOR);
+    assertToken(tokenList, 5, "$alertClass", CssTokenType.DOLLAR_IDENTIFIER);
+    assertToken(tokenList, 6, "}", CssTokenType.PUNCTUATOR);
+    assertToken(tokenList, 7, "{", CssTokenType.PUNCTUATOR);
+    assertToken(tokenList, 8, "color", CssTokenType.IDENTIFIER);
+    assertToken(tokenList, 9, ":", CssTokenType.PUNCTUATOR);
+    assertToken(tokenList, 10, "red", CssTokenType.IDENTIFIER);
+    assertToken(tokenList, 11, ";", CssTokenType.PUNCTUATOR);
+    assertToken(tokenList, 12, "}", CssTokenType.PUNCTUATOR);
   }
 
   @Test
-  public void scss_import() throws ScriptException {
-    List<Token> tokenList = tokenizer.tokenize("@import 'base';");
+  public void scss_import() {
+    List<CssToken> tokenList = tokenizer.tokenize("@import 'base';");
 
     assertThat(tokenList.size()).isEqualTo(3);
-    assertToken(tokenList, 0, "@import", Type.AT_WORD);
-    assertToken(tokenList, 1, "'base'", Type.STRING);
-    assertToken(tokenList, 2, ";", Type.PUNCTUATOR);
+    assertToken(tokenList, 0, "@import", CssTokenType.AT_IDENTIFIER);
+    assertToken(tokenList, 1, "'base'", CssTokenType.STRING);
+    assertToken(tokenList, 2, ";", CssTokenType.PUNCTUATOR);
   }
 
   @Test
-  public void scss_role() throws ScriptException {
-    List<Token> tokenList = tokenizer.tokenize("article[role=\"main\"] { width: 1px; }");
+  public void scss_role() {
+    List<CssToken> tokenList = tokenizer.tokenize("article[role=\"main\"] { width: 1px; }");
 
-    assertThat(tokenList.size()).isEqualTo(11);
-    assertToken(tokenList, 0, "article", Type.WORD);
-    assertToken(tokenList, 1, "[", Type.PUNCTUATOR);
-    assertToken(tokenList, 2, "role=", Type.WORD);
-    assertToken(tokenList, 3, "\"main\"", Type.STRING);
-    assertToken(tokenList, 4, "]", Type.PUNCTUATOR);
+    assertThat(tokenList.size()).isEqualTo(12);
+    assertToken(tokenList, 0, "article", CssTokenType.IDENTIFIER);
+    assertToken(tokenList, 1, "[", CssTokenType.PUNCTUATOR);
+    assertToken(tokenList, 2, "role", CssTokenType.IDENTIFIER);
+    assertToken(tokenList, 3, "=", CssTokenType.PUNCTUATOR);
+    assertToken(tokenList, 4, "\"main\"", CssTokenType.STRING);
+    assertToken(tokenList, 5, "]", CssTokenType.PUNCTUATOR);
   }
 
   @Test
-  public void scss_operators() throws ScriptException {
-    assertToken("foo { width: 300px + 960px; }", 5, "+", Type.WORD);
-    assertToken("foo { width: 300px - 960px; }", 5, "-", Type.WORD);
-    assertToken("foo { width: 300px * 960px; }", 5, "*", Type.WORD);
-    assertToken("foo { width: 300px / 960px; }", 5, "/", Type.WORD);
+  public void scss_less_operators() {
+    assertToken("foo { width: 300px + 960px; }", 5, "+", CssTokenType.PUNCTUATOR);
+    assertToken("foo { width: 300px - 960px; }", 5, "-", CssTokenType.PUNCTUATOR);
+    assertToken("foo { width: 300px * 960px; }", 5, "*", CssTokenType.PUNCTUATOR);
+    assertToken("foo { width: 300px / 960px; }", 5, "/", CssTokenType.PUNCTUATOR);
   }
 
   @Test
-  public void scss_parent_selector() throws ScriptException {
-    assertToken("a { &:hover { color: red; } }", 2, "&", Type.WORD);
-    assertToken("p { body.no-touch & { display: none; } }", 3, "&", Type.WORD);
+  public void scss_parent_selector() {
+    assertToken("a { &:hover { color: red; } }", 2, "&", CssTokenType.PUNCTUATOR);
+    assertToken("p { body.no-touch & { display: none; } }", 5, "&", CssTokenType.PUNCTUATOR);
   }
 
   @Test
-  public void scss_control_directives() throws ScriptException {
-    assertToken("@if ($debug) { }", 0, "@if", Type.AT_WORD);
-    assertToken("@each $name in 'save' 'cancel' { }", 0, "@each", Type.AT_WORD);
+  public void scss_control_directives() {
+    assertToken("@if ($debug) { }", 0, "@if", CssTokenType.AT_IDENTIFIER);
+    assertToken("@each $name in 'save' 'cancel' { }", 0, "@each", CssTokenType.AT_IDENTIFIER);
   }
 
   @Test
-  public void less_variable() throws ScriptException {
-    assertToken("@nice-blue: #5B83AD;", 0, "@nice-blue", Type.AT_WORD);
-    assertToken("foo { color: @@color; }", 4, "@@color", Type.AT_WORD);
+  public void less_variable() {
+    assertToken("@nice-blue: #5B83AD;", 0, "@nice-blue", CssTokenType.AT_IDENTIFIER);
+    assertToken("foo { color: @@color; }", 4, "@@color", CssTokenType.AT_IDENTIFIER);
   }
 
   @Test
-  public void less_operators() throws ScriptException {
-    assertToken("@base: 2cm * 3mm;", 3, "*", Type.WORD);
+  public void less_comment() {
+    assertToken("// Get in line!", 0, "// Get in line!", CssTokenType.COMMENT);
+    assertToken("// body font size = 62.5%\n\n/* some comment */", 0, "// body font size = 62.5%", CssTokenType.COMMENT);
+    assertToken("/* One heck of a block\n * style comment! */", 0, "/* One heck of a block\n * style comment! */", CssTokenType.COMMENT);
   }
 
   @Test
-  public void less_escaping() throws ScriptException {
-    assertToken("@min768: ~\"(min-width: 768px)\";", 2, "~", Type.WORD);
-  }
-
-  @Test
-  public void less_comment() throws ScriptException {
-    // FIXME: Less allows // comment which are not supported by our current tokenizer
-    //assertToken("// Get in line!", 0, "Get in line!", Type.COMMENT);
-
-    assertToken("/* One heck of a block\n * style comment! */", 0, "/* One heck of a block\n * style comment! */", Type.COMMENT);
+  public void unrecognized() {
+    assertToken("$$a", 0, "$a", CssTokenType.DOLLAR_IDENTIFIER);
   }
 
-  private static void assertToken(String input, int index, String value, Token.Type type) throws ScriptException {
-    List<Token> tokenList = tokenizer.tokenize(input);
-    assertToken(tokenList, index, value, type);
+  private static void assertToken(String input, int index, String value, CssTokenType CssTokenType) {
+    List<CssToken> tokenList = tokenizer.tokenize(input);
+    assertToken(tokenList, index, value, CssTokenType);
   }
 
-  private static void assertToken(String input, int index, String value, Token.Type type, int line, int column, int
-    endLine, int endColumn) throws ScriptException {
-    List<Token> tokenList = tokenizer.tokenize(input);
-    assertToken(tokenList, index, value, type, line, column, endLine, endColumn);
+  private static void assertToken(String input, int index, String value, CssTokenType CssTokenType, int line, int column, int endLine, int endColumn) {
+    List<CssToken> tokenList = tokenizer.tokenize(input);
+    assertToken(tokenList, index, value, CssTokenType, line, column, endLine, endColumn);
   }
 
-  private static void assertToken(List<Token> tokenList, int index, String value, Token.Type type) {
-    assertThat(tokenList.get(index).type).isEqualTo(type);
+  private static void assertToken(List<CssToken> tokenList, int index, String value, CssTokenType CssTokenType) {
+    assertThat(tokenList.get(index).type).isEqualTo(CssTokenType);
     assertThat(tokenList.get(index).text).isEqualTo(value);
   }
 
-  private static void assertToken(List<Token> tokenList, int index, String value, Token.Type type, int line, int column, int endLine, int endColumn) {
-    assertToken(tokenList, index, value, type);
+  private static void assertToken(List<CssToken> tokenList, int index, String value, CssTokenType CssTokenType, int line, int column, int endLine, int endColumn) {
+    assertToken(tokenList, index, value, CssTokenType);
     assertThat(tokenList.get(index).startLine).isEqualTo(line);
     assertThat(tokenList.get(index).startColumn).isEqualTo(column);
     assertThat(tokenList.get(index).endLine).isEqualTo(endLine);