Tokenize CSS (#40)

4 files changed, 782 insertions, 0 deletions

diff --git a/sonar-css-plugin/src/main/java/org/sonar/css/plugin/Token.java b/sonar-css-plugin/src/main/java/org/sonar/css/plugin/Token.java
new file mode 100644
index 0000000..dc9af61
--- /dev/null
+++ b/sonar-css-plugin/src/main/java/org/sonar/css/plugin/Token.java
@@ -0,0 +1,48 @@
+/*
+ * SonarCSS
+ * Copyright (C) 2018-2018 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.css.plugin;
+
+public class Token {
+
+  public enum Type {
+    COMMENT,
+    STRING,
+    WORD,
+    AT_WORD,
+    BRACKETS,
+    PUNCTUATOR
+  }
+
+  Type type;
+  String text;
+  Integer startLine;
+  Integer startColumn;
+  Integer endLine;
+  Integer endColumn;
+
+  public Token(Type type, String text, Integer startLine, Integer startColumn, Integer endLine, Integer endColumn) {
+    this.text = text;
+    this.type = type;
+    this.startLine = startLine;
+    this.startColumn = startColumn;
+    this.endLine = endLine;
+    this.endColumn = endColumn;
+  }
+}
diff --git a/sonar-css-plugin/src/main/java/org/sonar/css/plugin/Tokenizer.java b/sonar-css-plugin/src/main/java/org/sonar/css/plugin/Tokenizer.java
new file mode 100644
index 0000000..8f03492
--- /dev/null
+++ b/sonar-css-plugin/src/main/java/org/sonar/css/plugin/Tokenizer.java
@@ -0,0 +1,134 @@
+/*
+ * SonarCSS
+ * Copyright (C) 2018-2018 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.css.plugin;
+
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import javax.script.ScriptEngine;
+import javax.script.ScriptEngineManager;
+import javax.script.ScriptException;
+import org.sonar.api.internal.apachecommons.lang.StringEscapeUtils;
+import org.sonar.css.plugin.Token.Type;
+
+public class Tokenizer {
+
+  public List<Token> tokenize(String css) throws ScriptException {
+    ScriptEngineManager factory = new ScriptEngineManager();
+    ScriptEngine engine = factory.getEngineByName("JavaScript");
+    InputStream tokenizeScript = Tokenizer.class.getClassLoader().getResourceAsStream("tokenize.js");
+    engine.eval(new InputStreamReader(tokenizeScript, StandardCharsets.UTF_8));
+    String cssInput = "tokenize('" + StringEscapeUtils.escapeJavaScript(css) + "')";
+    Object tokens = engine.eval(cssInput);
+    return extractTokens(tokens);
+  }
+
+  private static List<Token> extractTokens(Object tokens) {
+    // tokens is result of call to javascript function tokenize(). It returns an array of arrays, where nested arrays
+    // correspond to tokens. These array javascript objects mapped in Java to Map objects where array index is key.
+
+    List<Token> resultList = new ArrayList<>();
+    for (Object tokenObject : ((Map<String, Object>) tokens).values()) {
+
+      // Access the inner arrays (disregard the keys) and use their length to decide which type of token we are
+      // dealing with.
+      Map<String, Object> tokenProperties = (Map<String, Object>) tokenObject;
+
+      // skip whitespace token (size < 4)
+      if (tokenProperties.size() >= 4) {
+        String text = tokenProperties.get("1").toString();
+        Type type = computeType(tokenProperties.get("0").toString(), text);
+        Integer startLine = convertToInt(tokenProperties.get("2"));
+        Integer startColumn = ((Double) tokenProperties.get("3")).intValue();
+
+        // all cases except for punctuator type
+        if (tokenProperties.size() == 6) {
+          Integer endLine = convertToInt(tokenProperties.get("4"));
+          Integer endColumn = ((Double) tokenProperties.get("5")).intValue();
+
+
+          if (isTokenWithPunctuator(text, ",", startLine, endLine)) {
+            resultList.addAll(splitTokenWithPunctuator(text, type, startLine, startColumn, endLine, endColumn));
+          } else if (isTokenWithPunctuator(text, ":", startLine, endLine)) {
+            resultList.addAll(splitTokenWithPunctuator(text, type, startLine, startColumn, endLine, endColumn));
+          } else {
+            resultList.add(new Token(type, text, startLine, startColumn, endLine, endColumn));
+          }
+        } else {
+          // is punctuator
+          resultList.add(new Token(type, text, startLine, startColumn, startLine, startColumn));
+        }
+      }
+    }
+
+    return resultList;
+  }
+
+  // Javascript tokenizer is not returning 2 tokens for words ending with a comma (e.g. foo,) and for words starting
+  // with at symbol and endings with colon (e.g. @base:) so we need to split the word into 2 tokens (1 word without
+  // the punctuator and 1 punctuator).
+  // For the sake of simplicity we don't handle words ending with the punctuator on a new line.
+  private static Boolean isTokenWithPunctuator(String text, String punctuator, Integer startLine, Integer endLine) {
+    return text.length() > 1 && text.endsWith(punctuator) && startLine.equals(endLine);
+  }
+
+  private static List<Token> splitTokenWithPunctuator(String text, Type type, Integer startLine, Integer startColumn, Integer endLine, Integer endColumn) {
+    List<Token> tokenList = new ArrayList<>();
+
+    tokenList.add(new Token(type, text.substring(0, text.length() - 1), startLine, startColumn, endLine, endColumn - 1));
+    tokenList.add(new Token(Type.PUNCTUATOR, text.substring(text.length() - 1), startLine, endColumn, endLine, endColumn));
+
+    return tokenList;
+  }
+
+  private static Integer convertToInt(Object value) {
+    if (value instanceof Double) {
+      return ((Double) value).intValue();
+    } else if (value instanceof Integer) {
+      return  (Integer) value;
+    } else {
+      throw new IllegalStateException("Failed to convert to number: " + value);
+    }
+  }
+
+  private static Type computeType(String type, String text) {
+    switch (type) {
+      case "at-word":
+        return Type.AT_WORD;
+      case "word":
+        if (",".equals(text)) {
+          return Type.PUNCTUATOR;
+        } else {
+          return Type.WORD;
+        }
+      case "comment":
+        return Type.COMMENT;
+      case "string":
+        return Type.STRING;
+      case "brackets":
+        return Type.BRACKETS;
+      default:
+        return Type.PUNCTUATOR;
+    }
+  }
+}
diff --git a/sonar-css-plugin/src/main/resources/tokenize.js b/sonar-css-plugin/src/main/resources/tokenize.js
new file mode 100644
index 0000000..bbf9b1b
--- /dev/null
+++ b/sonar-css-plugin/src/main/resources/tokenize.js
@@ -0,0 +1,374 @@
+/*
+ * SonarCSS
+ * Copyright (C) 2018-2018 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+
+/*
+ * The MIT License (MIT)
+ *
+ * Copyright 2013 Andrey Sitnik <andrey@sitnik.ru>
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy of
+ * this software and associated documentation files (the "Software"), to deal in
+ * the Software without restriction, including without limitation the rights to
+ * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+ * the Software, and to permit persons to whom the Software is furnished to do so,
+ * subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+ * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+ * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+ * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+var SINGLE_QUOTE      = '\''.charCodeAt(0);
+var DOUBLE_QUOTE      =  '"'.charCodeAt(0);
+var BACKSLASH         = '\\'.charCodeAt(0);
+var SLASH             =  '/'.charCodeAt(0);
+var NEWLINE           = '\n'.charCodeAt(0);
+var SPACE             =  ' '.charCodeAt(0);
+var FEED              = '\f'.charCodeAt(0);
+var TAB               = '\t'.charCodeAt(0);
+var CR                = '\r'.charCodeAt(0);
+var OPEN_SQUARE       =  '['.charCodeAt(0);
+var CLOSE_SQUARE      =  ']'.charCodeAt(0);
+var OPEN_PARENTHESES  =  '('.charCodeAt(0);
+var CLOSE_PARENTHESES =  ')'.charCodeAt(0);
+var OPEN_CURLY        =  '{'.charCodeAt(0);
+var CLOSE_CURLY       =  '}'.charCodeAt(0);
+var SEMICOLON         =  ';'.charCodeAt(0);
+var ASTERISK          =  '*'.charCodeAt(0);
+var COLON             =  ':'.charCodeAt(0);
+var AT                =  '@'.charCodeAt(0);
+
+var RE_AT_END      = /[ \n\t\r\f\{\(\)'"\\;/\[\]#]/g;
+var RE_WORD_END    = /[ \n\t\r\f\(\)\{\}:;@!'"\\\]\[#]|\/(?=\*)/g;
+var RE_BAD_BRACKET = /.[\\\/\("'\n]/;
+var RE_HEX_ESCAPE  = /[a-f0-9]/i;
+
+function tokenize(css) {
+	var input = {css: css, error: function() {}};
+	var processor = tokenizer(input);
+    var tokens = [];
+    while (!processor.endOfFile()) {
+        tokens.push(processor.nextToken());
+    }
+    return tokens;
+}
+
+function tokenizer(input ) {
+	var options = {};
+    var css = input.css.valueOf();
+    var ignore = options.ignoreErrors;
+
+    var code, next, quote, lines, last, content, escape,
+        nextLine, nextOffset, escaped, escapePos, prev, n, currentToken;
+
+    var length = css.length;
+    var offset = -1;
+    var line = 1;
+    var pos = 0;
+    var buffer = [];
+    var returned = [];
+
+    function unclosed(what) {
+        throw input.error('Unclosed ' + what, line, pos - offset);
+    }
+
+    function endOfFile() {
+        return returned.length === 0 && pos >= length;
+    }
+
+    function nextToken() {
+        if ( returned.length ) return returned.pop();
+        if ( pos >= length ) return;
+
+        code = css.charCodeAt(pos);
+        if ( code === NEWLINE || code === FEED ||
+             code === CR && css.charCodeAt(pos + 1) !== NEWLINE ) {
+            offset = pos;
+            line += 1;
+        }
+
+        switch ( code ) {
+        case NEWLINE:
+        case SPACE:
+        case TAB:
+        case CR:
+        case FEED:
+            next = pos;
+            do {
+                next += 1;
+                code = css.charCodeAt(next);
+                if ( code === NEWLINE ) {
+                    offset = next;
+                    line += 1;
+                }
+            } while ( code === SPACE   ||
+                      code === NEWLINE ||
+                      code === TAB     ||
+                      code === CR      ||
+                      code === FEED );
+
+            currentToken = ['space', css.slice(pos, next)];
+            pos = next - 1;
+            break;
+
+        case OPEN_SQUARE:
+            currentToken = ['[', '[', line, pos - offset];
+            break;
+
+        case CLOSE_SQUARE:
+            currentToken = [']', ']', line, pos - offset];
+            break;
+
+        case OPEN_CURLY:
+            currentToken = ['{', '{', line, pos - offset];
+            break;
+
+        case CLOSE_CURLY:
+            currentToken = ['}', '}', line, pos - offset];
+            break;
+
+        case COLON:
+            currentToken = [':', ':', line, pos - offset];
+            break;
+
+        case SEMICOLON:
+            currentToken = [';', ';', line, pos - offset];
+            break;
+
+        case OPEN_PARENTHESES:
+            prev = buffer.length ? buffer.pop()[1] : '';
+            n    = css.charCodeAt(pos + 1);
+            if ( prev === 'url' &&
+                 n !== SINGLE_QUOTE && n !== DOUBLE_QUOTE &&
+                 n !== SPACE && n !== NEWLINE && n !== TAB &&
+                 n !== FEED && n !== CR ) {
+                next = pos;
+                do {
+                    escaped = false;
+                    next    = css.indexOf(')', next + 1);
+                    if ( next === -1 ) {
+                        if ( ignore ) {
+                            next = pos;
+                            break;
+                        } else {
+                            unclosed('bracket');
+                        }
+                    }
+                    escapePos = next;
+                    while ( css.charCodeAt(escapePos - 1) === BACKSLASH ) {
+                        escapePos -= 1;
+                        escaped = !escaped;
+                    }
+                } while ( escaped );
+
+                currentToken = ['brackets', css.slice(pos, next + 1),
+                    line, pos  - offset,
+                    line, next - offset
+                ];
+
+                pos = next;
+
+            } else {
+                next    = css.indexOf(')', pos + 1);
+                content = css.slice(pos, next + 1);
+
+                if ( next === -1 || RE_BAD_BRACKET.test(content) ) {
+                    currentToken = ['(', '(', line, pos - offset];
+                } else {
+                    currentToken = ['brackets', content,
+                        line, pos  - offset,
+                        line, next - offset
+                    ];
+                    pos = next;
+                }
+            }
+
+            break;
+
+        case CLOSE_PARENTHESES:
+            currentToken = [')', ')', line, pos - offset];
+            break;
+
+        case SINGLE_QUOTE:
+        case DOUBLE_QUOTE:
+            quote = code === SINGLE_QUOTE ? '\'' : '"';
+            next  = pos;
+            do {
+                escaped = false;
+                next    = css.indexOf(quote, next + 1);
+                if ( next === -1 ) {
+                    if ( ignore ) {
+                        next = pos + 1;
+                        break;
+                    } else {
+                        unclosed('string');
+                    }
+                }
+                escapePos = next;
+                while ( css.charCodeAt(escapePos - 1) === BACKSLASH ) {
+                    escapePos -= 1;
+                    escaped = !escaped;
+                }
+            } while ( escaped );
+
+            content = css.slice(pos, next + 1);
+            lines   = content.split('\n');
+            last    = lines.length - 1;
+
+            if ( last > 0 ) {
+                nextLine   = line + last;
+                nextOffset = next - lines[last].length;
+            } else {
+                nextLine   = line;
+                nextOffset = offset;
+            }
+
+            currentToken = ['string', css.slice(pos, next + 1),
+                line, pos  - offset,
+                nextLine, next - nextOffset
+            ];
+
+            offset = nextOffset;
+            line   = nextLine;
+            pos    = next;
+            break;
+
+        case AT:
+            RE_AT_END.lastIndex = pos + 1;
+            RE_AT_END.test(css);
+            if ( RE_AT_END.lastIndex === 0 ) {
+                next = css.length - 1;
+            } else {
+                next = RE_AT_END.lastIndex - 2;
+            }
+
+            currentToken = ['at-word', css.slice(pos, next + 1),
+                line, pos  - offset,
+                line, next - offset
+            ];
+
+            pos = next;
+            break;
+
+        case BACKSLASH:
+            next   = pos;
+            escape = true;
+            while ( css.charCodeAt(next + 1) === BACKSLASH ) {
+                next  += 1;
+                escape = !escape;
+            }
+            code = css.charCodeAt(next + 1);
+            if ( escape && (code !== SLASH   &&
+                            code !== SPACE   &&
+                            code !== NEWLINE &&
+                            code !== TAB     &&
+                            code !== CR      &&
+                            code !== FEED ) ) {
+                next += 1;
+                if ( RE_HEX_ESCAPE.test(css.charAt(next)) ) {
+                    while ( RE_HEX_ESCAPE.test(css.charAt(next + 1)) ) {
+                        next += 1;
+                    }
+                    if ( css.charCodeAt(next + 1) === SPACE ) {
+                        next += 1;
+                    }
+                }
+            }
+
+            currentToken = ['word', css.slice(pos, next + 1),
+                line, pos  - offset,
+                line, next - offset
+            ];
+
+            pos = next;
+            break;
+
+        default:
+            if ( code === SLASH && css.charCodeAt(pos + 1) === ASTERISK ) {
+                next = css.indexOf('*/', pos + 2) + 1;
+                if ( next === 0 ) {
+                    if ( ignore ) {
+                        next = css.length;
+                    } else {
+                        unclosed('comment');
+                    }
+                }
+
+                content = css.slice(pos, next + 1);
+                lines   = content.split('\n');
+                last    = lines.length - 1;
+
+                if ( last > 0 ) {
+                    nextLine   = line + last;
+                    nextOffset = next - lines[last].length;
+                } else {
+                    nextLine   = line;
+                    nextOffset = offset;
+                }
+
+                currentToken = ['comment', content,
+                    line,     pos  - offset,
+                    nextLine, next - nextOffset
+                ];
+
+                offset = nextOffset;
+                line   = nextLine;
+                pos    = next;
+
+            } else {
+                RE_WORD_END.lastIndex = pos + 1;
+                RE_WORD_END.test(css);
+                if ( RE_WORD_END.lastIndex === 0 ) {
+                    next = css.length - 1;
+                } else {
+                    next = RE_WORD_END.lastIndex - 2;
+                }
+
+                currentToken = ['word', css.slice(pos, next + 1),
+                    line, pos  - offset,
+                    line, next - offset
+                ];
+
+                buffer.push(currentToken);
+
+                pos = next;
+            }
+
+            break;
+        }
+
+        pos++;
+        return currentToken;
+    }
+
+    function back(token) {
+        returned.push(token);
+    }
+
+    return {
+        back:back, nextToken:nextToken, endOfFile:endOfFile
+    };
+}
diff --git a/sonar-css-plugin/src/test/java/org/sonar/css/plugin/TokenizerTest.java b/sonar-css-plugin/src/test/java/org/sonar/css/plugin/TokenizerTest.java
new file mode 100644
index 0000000..b1589c0
--- /dev/null
+++ b/sonar-css-plugin/src/test/java/org/sonar/css/plugin/TokenizerTest.java
@@ -0,0 +1,226 @@
+/*
+ * SonarCSS
+ * Copyright (C) 2018-2018 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.css.plugin;
+
+import java.util.List;
+import java.util.Optional;
+
+import javax.script.ScriptException;
+import org.junit.Test;
+import org.sonar.css.plugin.Token.Type;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+public class TokenizerTest {
+
+  private final static Tokenizer tokenizer = new Tokenizer();
+  
+  @Test
+  public void word() throws ScriptException {
+    assertToken("bar { }", 0, "bar", Type.WORD);
+    assertToken("bar: foo { }", 0, "bar", Type.WORD);
+    assertToken("bar: foo-baz { }", 2, "foo-baz", Type.WORD);
+    assertToken("foo bar { }", 1, "bar", Type.WORD);
+    assertToken("#bar { }", 0, "#bar", Type.WORD);
+    assertToken("foo.bar { }", 0, "foo.bar", Type.WORD);
+    assertToken(".bar { }", 0, ".bar", Type.WORD);
+    assertToken("bar { foo: 42; }", 2, "foo", Type.WORD);
+    assertToken("bar { foo: baz; }", 4, "baz", Type.WORD);
+    assertToken("foo , bar { }", 2, "bar", Type.WORD);
+  }
+
+  @Test
+  public void semi_colon() throws ScriptException {
+    assertToken("bar: foo { }", 1, ":", Type.PUNCTUATOR);
+    assertToken("bar { foo; }", 3, ";", Type.PUNCTUATOR);
+  }
+
+  @Test
+  public void comma() throws ScriptException {
+    assertToken("foo , bar { }", 1, ",", Type.PUNCTUATOR);
+    assertToken("foo, bar { }", 1, ",", Type.PUNCTUATOR);
+  }
+
+  @Test
+  public void number_as_word() throws ScriptException {
+    assertToken("bar { foo: 1.15; }", 4, "1.15", Type.WORD);
+    assertToken("bar { foo: 1; }", 4, "1", Type.WORD);
+    assertToken("bar { foo: 1.15px; }", 4, "1.15px", Type.WORD);
+    assertToken("bar { foo: 1.15%; }", 4, "1.15%", Type.WORD);
+    assertToken("bar { foo: 1px; }", 4, "1px", Type.WORD);
+    assertToken("bar { foo: 1em/150%; }", 4, "1em/150%", Type.WORD);
+  }
+
+  @Test
+  public void brackets() throws ScriptException {
+    assertToken("bar { foo: (1.15); }", 4, "(1.15)", Type.BRACKETS);
+    assertToken("bar { foo: ( 1.15 ); }", 4, "( 1.15 )", Type.BRACKETS);
+    assertToken("bar { foo: (1.15 1 0px); }", 4, "(1.15 1 0px)", Type.BRACKETS);
+    assertToken("bar { foo: (1.15, 1, 0px); }", 4, "(1.15, 1, 0px)", Type.BRACKETS);
+    assertToken("bar { content: string(doctitle); }", 5, "(doctitle)", Type.BRACKETS);
+    assertToken("bar { string-set: booktitle content(); }", 6, "()", Type.BRACKETS);
+    assertToken("bar { a: b(attr(href, url), c) \")\"; }", 7, "(href, url)", Type.BRACKETS);
+  }
+
+  @Test
+  public void strings() throws ScriptException {
+    assertToken("bar { foo: \"\"; }", 4, "\"\"", Type.STRING);
+    assertToken("bar { foo: \"hello, world\"; }", 4, "\"hello, world\"", Type.STRING);
+  }
+
+  @Test
+  public void at_word() throws ScriptException {
+    assertToken("@bar { }", 0, "@bar", Type.AT_WORD);
+  }
+
+  @Test
+  public void comment() throws ScriptException {
+    assertToken("/* foo */", 0, "/* foo */", Type.COMMENT);
+    assertToken("foo { a: /* foo */ 42; }", 4, "/* foo */", Type.COMMENT);
+    assertToken("/* \n"
+      + "  this is a comment\n"
+      + "  and it is awesome because\n"
+      + "  it is multiline!\n"
+      + "*/", 0, "/* \n"
+      + "  this is a comment\n"
+      + "  and it is awesome because\n"
+      + "  it is multiline!\n"
+      + "*/", Type.COMMENT, 1, 1, 5, 2);
+    assertToken("foo { a: /* foo\nbar*/ 42; }", 4, "/* foo\nbar*/", Type.COMMENT, 1, 10, 2, 5);
+  }
+
+  @Test
+  public void hashtag() throws ScriptException {
+    assertToken("bar { color: #333; }", 4, "#333", Type.WORD);
+    assertToken("bar { color: #e535ab; }", 4, "#e535ab", Type.WORD);
+    assertToken("#table-of-contents + ul li { list-style: none; }", 0, "#table-of-contents", Type.WORD);
+  }
+
+  @Test
+  public void scss_variable() throws ScriptException {
+    assertToken("$font-stack: Helvetica;", 0, "$font-stack", Type.WORD);
+    assertToken("$message-color: blue !default;", 3, "!default", Type.WORD);
+
+    List<Token> tokenList = tokenizer.tokenize("p.message-#{$alertClass} { color: red; }");
+    assertThat(tokenList.size()).isEqualTo(11);
+    assertToken(tokenList, 0, "p.message-", Type.WORD);
+    assertToken(tokenList, 1, "#", Type.WORD);
+    assertToken(tokenList, 2, "{", Type.PUNCTUATOR);
+    assertToken(tokenList, 3, "$alertClass", Type.WORD);
+    assertToken(tokenList, 4, "}", Type.PUNCTUATOR);
+    assertToken(tokenList, 5, "{", Type.PUNCTUATOR);
+    assertToken(tokenList, 6, "color", Type.WORD);
+    assertToken(tokenList, 7, ":", Type.PUNCTUATOR);
+    assertToken(tokenList, 8, "red", Type.WORD);
+    assertToken(tokenList, 9, ";", Type.PUNCTUATOR);
+    assertToken(tokenList, 10, "}", Type.PUNCTUATOR);
+
+  }
+
+  @Test
+  public void scss_import() throws ScriptException {
+    List<Token> tokenList = tokenizer.tokenize("@import 'base';");
+
+    assertThat(tokenList.size()).isEqualTo(3);
+    assertToken(tokenList, 0, "@import", Type.AT_WORD);
+    assertToken(tokenList, 1, "'base'", Type.STRING);
+    assertToken(tokenList, 2, ";", Type.PUNCTUATOR);
+  }
+
+  @Test
+  public void scss_role() throws ScriptException {
+    List<Token> tokenList = tokenizer.tokenize("article[role=\"main\"] { width: 1px; }");
+
+    assertThat(tokenList.size()).isEqualTo(11);
+    assertToken(tokenList, 0, "article", Type.WORD);
+    assertToken(tokenList, 1, "[", Type.PUNCTUATOR);
+    assertToken(tokenList, 2, "role=", Type.WORD);
+    assertToken(tokenList, 3, "\"main\"", Type.STRING);
+    assertToken(tokenList, 4, "]", Type.PUNCTUATOR);
+  }
+
+  @Test
+  public void scss_operators() throws ScriptException {
+    assertToken("foo { width: 300px + 960px; }", 5, "+", Type.WORD);
+    assertToken("foo { width: 300px - 960px; }", 5, "-", Type.WORD);
+    assertToken("foo { width: 300px * 960px; }", 5, "*", Type.WORD);
+    assertToken("foo { width: 300px / 960px; }", 5, "/", Type.WORD);
+  }
+
+  @Test
+  public void scss_parent_selector() throws ScriptException {
+    assertToken("a { &:hover { color: red; } }", 2, "&", Type.WORD);
+    assertToken("p { body.no-touch & { display: none; } }", 3, "&", Type.WORD);
+  }
+
+  @Test
+  public void scss_control_directives() throws ScriptException {
+    assertToken("@if ($debug) { }", 0, "@if", Type.AT_WORD);
+    assertToken("@each $name in 'save' 'cancel' { }", 0, "@each", Type.AT_WORD);
+  }
+
+  @Test
+  public void less_variable() throws ScriptException {
+    assertToken("@nice-blue: #5B83AD;", 0, "@nice-blue", Type.AT_WORD);
+    assertToken("foo { color: @@color; }", 4, "@@color", Type.AT_WORD);
+  }
+
+  @Test
+  public void less_operators() throws ScriptException {
+    assertToken("@base: 2cm * 3mm;", 3, "*", Type.WORD);
+  }
+
+  @Test
+  public void less_escaping() throws ScriptException {
+    assertToken("@min768: ~\"(min-width: 768px)\";", 2, "~", Type.WORD);
+  }
+
+  @Test
+  public void less_comment() throws ScriptException {
+    // FIXME: Less allows // comment which are not supported by our current tokenizer
+    //assertToken("// Get in line!", 0, "Get in line!", Type.COMMENT);
+
+    assertToken("/* One heck of a block\n * style comment! */", 0, "/* One heck of a block\n * style comment! */", Type.COMMENT);
+  }
+
+  private static void assertToken(String input, int index, String value, Token.Type type) throws ScriptException {
+    List<Token> tokenList = tokenizer.tokenize(input);
+    assertToken(tokenList, index, value, type);
+  }
+
+  private static void assertToken(String input, int index, String value, Token.Type type, int line, int column, int
+    endLine, int endColumn) throws ScriptException {
+    List<Token> tokenList = tokenizer.tokenize(input);
+    assertToken(tokenList, index, value, type, line, column, endLine, endColumn);
+  }
+
+  private static void assertToken(List<Token> tokenList, int index, String value, Token.Type type) {
+    assertThat(tokenList.get(index).type).isEqualTo(type);
+    assertThat(tokenList.get(index).text).isEqualTo(value);
+  }
+
+  private static void assertToken(List<Token> tokenList, int index, String value, Token.Type type, int line, int column, int endLine, int endColumn) {
+    assertToken(tokenList, index, value, type);
+    assertThat(tokenList.get(index).startLine).isEqualTo(line);
+    assertThat(tokenList.get(index).startColumn).isEqualTo(column);
+    assertThat(tokenList.get(index).endLine).isEqualTo(endLine);
+    assertThat(tokenList.get(index).endColumn).isEqualTo(endColumn);
+  }
+}