aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--.gitignore3
-rw-r--r--pom.xml14
-rw-r--r--sonar-css-plugin/pom.xml7
-rw-r--r--sonar-css-plugin/src/main/java/org/sonar/css/plugin/CssLexer.java76
-rw-r--r--sonar-css-plugin/src/main/java/org/sonar/css/plugin/CssToken.java45
-rw-r--r--sonar-css-plugin/src/main/java/org/sonar/css/plugin/CssTokenType.java (renamed from sonar-css-plugin/src/main/java/org/sonar/css/plugin/Token.java)43
-rw-r--r--sonar-css-plugin/src/main/java/org/sonar/css/plugin/MetricSensor.java39
-rw-r--r--sonar-css-plugin/src/main/java/org/sonar/css/plugin/Tokenizer.java171
-rw-r--r--sonar-css-plugin/src/main/resources/tokenize.js374
-rw-r--r--sonar-css-plugin/src/test/java/org/sonar/css/plugin/MetricSensorTest.java15
-rw-r--r--sonar-css-plugin/src/test/java/org/sonar/css/plugin/TokenizerTest.java241
11 files changed, 350 insertions, 678 deletions
diff --git a/.gitignore b/.gitignore
index c4f541d..e02da25 100644
--- a/.gitignore
+++ b/.gitignore
@@ -40,3 +40,6 @@ target/
# npm
node_modules/
+
+# Visual Studio
+.vs/
diff --git a/pom.xml b/pom.xml
index e4d7bff..eed37f0 100644
--- a/pom.xml
+++ b/pom.xml
@@ -74,7 +74,7 @@
<slf4j.version>1.7.25</slf4j.version>
<assertj.version>3.10.0</assertj.version>
<junit.version>4.12</junit.version>
- <sonar.version>7.2.0.13185</sonar.version>
+ <sonar.version>7.2</sonar.version>
<sonar.min.version>6.7</sonar.min.version>
<sonar-orchestrator.version>3.17.0.1491</sonar-orchestrator.version>
<sonarlint.version>3.1.0.1376</sonarlint.version>
@@ -83,6 +83,8 @@
<commons-io.version>2.5</commons-io.version>
<gson.version>2.6.2</gson.version>
<sonar-analyzer-commons.version>1.7.0.243</sonar-analyzer-commons.version>
+ <sslr.version>1.22</sslr.version>
+ <commonslang.version>2.6</commonslang.version>
<artifactsToPublish>${project.groupId}:sonar-css-plugin:jar</artifactsToPublish>
</properties>
@@ -101,6 +103,16 @@
<version>${sonar-analyzer-commons.version}</version>
</dependency>
<dependency>
+ <groupId>commons-lang</groupId>
+ <artifactId>commons-lang</artifactId>
+ <version>${commonslang.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.sonarsource.sslr</groupId>
+ <artifactId>sslr-core</artifactId>
+ <version>${sslr.version}</version>
+ </dependency>
+ <dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
<version>${commons-io.version}</version>
diff --git a/sonar-css-plugin/pom.xml b/sonar-css-plugin/pom.xml
index ee91ea8..0ad715c 100644
--- a/sonar-css-plugin/pom.xml
+++ b/sonar-css-plugin/pom.xml
@@ -59,7 +59,10 @@
<dependency>
<groupId>commons-lang</groupId>
<artifactId>commons-lang</artifactId>
- <version>2.6</version>
+ </dependency>
+ <dependency>
+ <groupId>org.sonarsource.sslr</groupId>
+ <artifactId>sslr-core</artifactId>
</dependency>
<dependency>
<groupId>commons-io</groupId>
@@ -124,7 +127,7 @@
<rules>
<requireFilesSize>
<minsize>5000000</minsize>
- <maxsize>6000000</maxsize>
+ <maxsize>6100000</maxsize>
<files>
<file>${project.build.directory}/${project.build.finalName}.jar</file>
</files>
diff --git a/sonar-css-plugin/src/main/java/org/sonar/css/plugin/CssLexer.java b/sonar-css-plugin/src/main/java/org/sonar/css/plugin/CssLexer.java
new file mode 100644
index 0000000..9a4bb58
--- /dev/null
+++ b/sonar-css-plugin/src/main/java/org/sonar/css/plugin/CssLexer.java
@@ -0,0 +1,76 @@
+/*
+ * SonarCSS
+ * Copyright (C) 2018-2018 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+package org.sonar.css.plugin;
+
+import com.sonar.sslr.impl.Lexer;
+
+import static com.sonar.sslr.impl.channel.RegexpChannelBuilder.regexp;
+
+// This is a at-best lexer.
+// It is far from being entirely matching the standard definition of css/less/scss tokens nor
+// following the theory of what a lexer responsibilities are but as we are only building line metrics and highlighting
+// on top of it we decided to focus on simplicity over being extensive.
+public final class CssLexer {
+
+ private static final String NEW_LINE = "(\r\n|\r|\n|\f)";
+ private static final String WHITESPACE = "[\t\n\f\r ]";
+ private static final String NON_ASCII = "[^\\p{ASCII}]";
+ private static final String HEX_DIGIT = "0-9a-fA-F";
+ private static final String ESCAPE = "(\\\\[" + HEX_DIGIT + "]{1,6}" + WHITESPACE + "?)|\\[^\r\n\f" + HEX_DIGIT + "]";
+
+ private static final String PUNCTUATOR = "[!:,;%&+#\\*-/=>\\(\\)\\[\\]\\{\\}]";
+
+ private static final String MULTI_LINE_COMMENT = "/\\*(.|" + NEW_LINE + ")*?\\*/";
+ private static final String INLINE_COMMENT = "//.*";
+ private static final String COMMENT = "(" + INLINE_COMMENT + "|" + MULTI_LINE_COMMENT + ")";
+
+ private static final String NUMBER = "[+|-]?\\d*\\.?\\d+([a-z]+|%)?";
+
+ private static final String NAME_CHAR = "[a-zA-Z0-9_-]|" + NON_ASCII + "|" + ESCAPE;
+ private static final String NAME_START = "[a-zA-Z_]|" + NON_ASCII + "|" + ESCAPE;
+
+ private static final String IDENTIFIER = "-?(" + NAME_START + ")(" + NAME_CHAR + ")*";
+ private static final String AT_IDENTIFIER = "@+" + IDENTIFIER;
+ private static final String HASH_IDENTIFIER = "#(" + NAME_CHAR + ")+";
+ private static final String DOLLAR_IDENTIFIER = "\\$(" + NAME_CHAR + ")+";
+
+ private static final String DOUBLE_QUOTE_STRING = "~?\"([^\"\\\\\r\n\f]|" + ESCAPE + "|\\\\" + NEW_LINE + ")*\"";
+ private static final String SINGLE_QUOTE_STRING = "~?'([^'\\\\\r\n\f]|" + ESCAPE + "|\\\\" + NEW_LINE + ")*'";
+ private static final String STRING = "(" + SINGLE_QUOTE_STRING + "|" + DOUBLE_QUOTE_STRING + ")";
+
+ private CssLexer() {
+ }
+
+ public static Lexer create() {
+ return Lexer.builder()
+ .withFailIfNoChannelToConsumeOneCharacter(false)
+
+ .withChannel(regexp(CssTokenType.COMMENT, COMMENT))
+ .withChannel(regexp(CssTokenType.STRING, STRING))
+ .withChannel(regexp(CssTokenType.AT_IDENTIFIER, AT_IDENTIFIER))
+ .withChannel(regexp(CssTokenType.HASH_IDENTIFIER, HASH_IDENTIFIER))
+ .withChannel(regexp(CssTokenType.DOLLAR_IDENTIFIER, DOLLAR_IDENTIFIER))
+ .withChannel(regexp(CssTokenType.IDENTIFIER, IDENTIFIER))
+ .withChannel(regexp(CssTokenType.NUMBER, NUMBER))
+ .withChannel(regexp(CssTokenType.PUNCTUATOR, PUNCTUATOR))
+
+ .build();
+ }
+}
diff --git a/sonar-css-plugin/src/main/java/org/sonar/css/plugin/CssToken.java b/sonar-css-plugin/src/main/java/org/sonar/css/plugin/CssToken.java
new file mode 100644
index 0000000..4ba0cc6
--- /dev/null
+++ b/sonar-css-plugin/src/main/java/org/sonar/css/plugin/CssToken.java
@@ -0,0 +1,45 @@
+/*
+ * SonarCSS
+ * Copyright (C) 2018-2018 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+package org.sonar.css.plugin;
+
+import com.sonar.sslr.api.Token;
+import com.sonar.sslr.api.TokenType;
+import org.sonarsource.analyzer.commons.TokenLocation;
+
+public class CssToken {
+ CssTokenType type;
+ String text;
+ Integer startLine;
+ Integer startColumn;
+ Integer endLine;
+ Integer endColumn;
+
+ public CssToken(Token token) {
+ TokenType tokenType = token.getType();
+ this.type = (CssTokenType)tokenType;
+ this.text = token.getValue();
+
+ TokenLocation tokenLocation = new TokenLocation(token.getLine(), token.getColumn(), token.getValue());
+ this.startLine = tokenLocation.startLine();
+ this.startColumn = tokenLocation.startLineOffset();
+ this.endLine = tokenLocation.endLine();
+ this.endColumn = tokenLocation.endLineOffset();
+ }
+}
diff --git a/sonar-css-plugin/src/main/java/org/sonar/css/plugin/Token.java b/sonar-css-plugin/src/main/java/org/sonar/css/plugin/CssTokenType.java
index dc9af61..dccc5b7 100644
--- a/sonar-css-plugin/src/main/java/org/sonar/css/plugin/Token.java
+++ b/sonar-css-plugin/src/main/java/org/sonar/css/plugin/CssTokenType.java
@@ -19,30 +19,31 @@
*/
package org.sonar.css.plugin;
-public class Token {
+import com.sonar.sslr.api.AstNode;
+import com.sonar.sslr.api.TokenType;
- public enum Type {
- COMMENT,
- STRING,
- WORD,
- AT_WORD,
- BRACKETS,
- PUNCTUATOR
+public enum CssTokenType implements TokenType {
+ COMMENT,
+ PUNCTUATOR,
+ NUMBER,
+ STRING,
+ AT_IDENTIFIER,
+ HASH_IDENTIFIER,
+ DOLLAR_IDENTIFIER,
+ IDENTIFIER;
+
+ @Override
+ public String getName() {
+ return name();
}
- Type type;
- String text;
- Integer startLine;
- Integer startColumn;
- Integer endLine;
- Integer endColumn;
+ @Override
+ public String getValue() {
+ return name();
+ }
- public Token(Type type, String text, Integer startLine, Integer startColumn, Integer endLine, Integer endColumn) {
- this.text = text;
- this.type = type;
- this.startLine = startLine;
- this.startColumn = startColumn;
- this.endLine = endLine;
- this.endColumn = endColumn;
+ @Override
+ public boolean hasToBeSkippedFromAst(AstNode node) {
+ return false;
}
}
diff --git a/sonar-css-plugin/src/main/java/org/sonar/css/plugin/MetricSensor.java b/sonar-css-plugin/src/main/java/org/sonar/css/plugin/MetricSensor.java
index abbbd50..6257b74 100644
--- a/sonar-css-plugin/src/main/java/org/sonar/css/plugin/MetricSensor.java
+++ b/sonar-css-plugin/src/main/java/org/sonar/css/plugin/MetricSensor.java
@@ -21,7 +21,6 @@ package org.sonar.css.plugin;
import java.io.IOException;
import java.util.List;
-import javax.script.ScriptException;
import org.sonar.api.batch.fs.FileSystem;
import org.sonar.api.batch.fs.InputFile;
import org.sonar.api.batch.sensor.Sensor;
@@ -57,11 +56,11 @@ public class MetricSensor implements Sensor {
private static void saveHighlights(SensorContext sensorContext, InputFile input, Tokenizer tokenizer) {
try {
NewHighlighting highlighting = sensorContext.newHighlighting().onFile(input);
- List<Token> tokenList = tokenizer.tokenize(input.contents());
+ List<CssToken> tokenList = tokenizer.tokenize(input.contents());
for (int i = 0; i < tokenList.size(); i++) {
- Token currentToken = tokenList.get(i);
- Token nextToken = i + 1 == tokenList.size() ? null : tokenList.get(i + 1);
+ CssToken currentToken = tokenList.get(i);
+ CssToken nextToken = i + 1 < tokenList.size() ? tokenList.get(i + 1) : null;
TypeOfText highlightingType = null;
switch (currentToken.type) {
@@ -73,18 +72,32 @@ public class MetricSensor implements Sensor {
highlightingType = TypeOfText.STRING;
break;
- case WORD:
- if (Character.isDigit(currentToken.text.charAt(0)) || currentToken.text.matches("^#[0-9a-fA-F]+$")) {
+ case NUMBER:
+ highlightingType = TypeOfText.CONSTANT;
+ break;
+
+ case AT_IDENTIFIER:
+ highlightingType = TypeOfText.ANNOTATION;
+ break;
+
+ case DOLLAR_IDENTIFIER:
+ highlightingType = TypeOfText.KEYWORD;
+ break;
+
+ case HASH_IDENTIFIER:
+ if (currentToken.text.matches("^#[0-9a-fA-F]+$")) {
highlightingType = TypeOfText.CONSTANT;
- } else if (nextToken != null && nextToken.text.equals(":")) {
- highlightingType = TypeOfText.KEYWORD_LIGHT;
- } else if (currentToken.text.startsWith(".") || (nextToken != null && nextToken.text.startsWith("{"))) {
+ } else {
highlightingType = TypeOfText.KEYWORD;
}
break;
- case AT_WORD:
- highlightingType = TypeOfText.ANNOTATION;
+ case IDENTIFIER:
+ // We want to highlight the property key of a css/scss/less file and as the tokenizer is putting the ':' into another token
+ // we need to look for identifier followed by a PUNCTUATOR token with text ':'.
+ if (nextToken != null && nextToken.text.equals(":")) {
+ highlightingType = TypeOfText.KEYWORD_LIGHT;
+ }
break;
default:
@@ -92,14 +105,12 @@ public class MetricSensor implements Sensor {
}
if (highlightingType != null) {
- highlighting.highlight(currentToken.startLine, currentToken.startColumn - 1, currentToken.endLine, currentToken.endColumn, highlightingType);
+ highlighting.highlight(currentToken.startLine, currentToken.startColumn, currentToken.endLine, currentToken.endColumn, highlightingType);
}
}
highlighting.save();
- } catch (ScriptException e) {
- LOG.error(String.format("Failed to tokenize file '%s'", input.toString()), e);
} catch (IOException e) {
LOG.error(String.format("Failed to read file '%s'", input.toString()), e);
}
diff --git a/sonar-css-plugin/src/main/java/org/sonar/css/plugin/Tokenizer.java b/sonar-css-plugin/src/main/java/org/sonar/css/plugin/Tokenizer.java
index 220bfaa..cf84e08 100644
--- a/sonar-css-plugin/src/main/java/org/sonar/css/plugin/Tokenizer.java
+++ b/sonar-css-plugin/src/main/java/org/sonar/css/plugin/Tokenizer.java
@@ -1,133 +1,38 @@
-/*
- * SonarCSS
- * Copyright (C) 2018-2018 SonarSource SA
- * mailto:info AT sonarsource DOT com
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 3 of the License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public License
- * along with this program; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
- */
-package org.sonar.css.plugin;
-
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.nio.charset.StandardCharsets;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-import javax.script.ScriptEngine;
-import javax.script.ScriptException;
-import jdk.nashorn.api.scripting.NashornScriptEngineFactory;
-import org.apache.commons.lang.StringEscapeUtils;
-import org.sonar.css.plugin.Token.Type;
-
-public class Tokenizer {
-
- public List<Token> tokenize(String css) throws ScriptException {
- ScriptEngine engine = new NashornScriptEngineFactory().getScriptEngine();
- InputStream tokenizeScript = Tokenizer.class.getClassLoader().getResourceAsStream("tokenize.js");
- engine.eval(new InputStreamReader(tokenizeScript, StandardCharsets.UTF_8));
- String cssInput = "tokenize('" + StringEscapeUtils.escapeJavaScript(css) + "')";
- Object tokens = engine.eval(cssInput);
- return extractTokens(tokens);
- }
-
- private static List<Token> extractTokens(Object tokens) {
- // tokens is result of call to javascript function tokenize(). It returns an array of arrays, where nested arrays
- // correspond to tokens. These array javascript objects mapped in Java to Map objects where array index is key.
-
- List<Token> resultList = new ArrayList<>();
- for (Object tokenObject : ((Map<String, Object>) tokens).values()) {
-
- // Access the inner arrays (disregard the keys) and use their length to decide which type of token we are
- // dealing with.
- Map<String, Object> tokenProperties = (Map<String, Object>) tokenObject;
-
- // skip whitespace token (size < 4)
- if (tokenProperties.size() >= 4) {
- String text = tokenProperties.get("1").toString();
- Type type = computeType(tokenProperties.get("0").toString(), text);
- Integer startLine = convertToInt(tokenProperties.get("2"));
- Integer startColumn = ((Double) tokenProperties.get("3")).intValue();
-
- // all cases except for punctuator type
- if (tokenProperties.size() == 6) {
- Integer endLine = convertToInt(tokenProperties.get("4"));
- Integer endColumn = ((Double) tokenProperties.get("5")).intValue();
-
-
- if (isTokenWithPunctuator(text, ",", startLine, endLine)) {
- resultList.addAll(splitTokenWithPunctuator(text, type, startLine, startColumn, endLine, endColumn));
- } else if (isTokenWithPunctuator(text, ":", startLine, endLine)) {
- resultList.addAll(splitTokenWithPunctuator(text, type, startLine, startColumn, endLine, endColumn));
- } else {
- resultList.add(new Token(type, text, startLine, startColumn, endLine, endColumn));
- }
- } else {
- // is punctuator
- resultList.add(new Token(type, text, startLine, startColumn, startLine, startColumn));
- }
- }
- }
-
- return resultList;
- }
-
- // Javascript tokenizer is not returning 2 tokens for words ending with a comma (e.g. foo,) and for words starting
- // with at symbol and endings with colon (e.g. @base:) so we need to split the word into 2 tokens (1 word without
- // the punctuator and 1 punctuator).
- // For the sake of simplicity we don't handle words ending with the punctuator on a new line.
- private static Boolean isTokenWithPunctuator(String text, String punctuator, Integer startLine, Integer endLine) {
- return text.length() > 1 && text.endsWith(punctuator) && startLine.equals(endLine);
- }
-
- private static List<Token> splitTokenWithPunctuator(String text, Type type, Integer startLine, Integer startColumn, Integer endLine, Integer endColumn) {
- List<Token> tokenList = new ArrayList<>();
-
- tokenList.add(new Token(type, text.substring(0, text.length() - 1), startLine, startColumn, endLine, endColumn - 1));
- tokenList.add(new Token(Type.PUNCTUATOR, text.substring(text.length() - 1), startLine, endColumn, endLine, endColumn));
-
- return tokenList;
- }
-
- private static Integer convertToInt(Object value) {
- if (value instanceof Double) {
- return ((Double) value).intValue();
- } else if (value instanceof Integer) {
- return (Integer) value;
- } else {
- throw new IllegalStateException("Failed to convert to number: " + value);
- }
- }
-
- private static Type computeType(String type, String text) {
- switch (type) {
- case "at-word":
- return Type.AT_WORD;
- case "word":
- if (",".equals(text)) {
- return Type.PUNCTUATOR;
- } else {
- return Type.WORD;
- }
- case "comment":
- return Type.COMMENT;
- case "string":
- return Type.STRING;
- case "brackets":
- return Type.BRACKETS;
- default:
- return Type.PUNCTUATOR;
- }
- }
-}
+/*
+ * SonarCSS
+ * Copyright (C) 2018-2018 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+package org.sonar.css.plugin;
+
+import com.sonar.sslr.api.Token;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.stream.Collectors;
+
+public class Tokenizer {
+
+ public List<CssToken> tokenize(String css) {
+ List<Token> tokenList = CssLexer.create().lex(css);
+
+ // remove last token (EOF token)
+ List<Token> cloneTokenList = new ArrayList<>(tokenList);
+ cloneTokenList.remove(cloneTokenList.size() - 1);
+
+ return cloneTokenList.stream().map(CssToken::new).collect(Collectors.toList());
+ }
+}
diff --git a/sonar-css-plugin/src/main/resources/tokenize.js b/sonar-css-plugin/src/main/resources/tokenize.js
deleted file mode 100644
index bbf9b1b..0000000
--- a/sonar-css-plugin/src/main/resources/tokenize.js
+++ /dev/null
@@ -1,374 +0,0 @@
-/*
- * SonarCSS
- * Copyright (C) 2018-2018 SonarSource SA
- * mailto:info AT sonarsource DOT com
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 3 of the License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public License
- * along with this program; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
- */
-
-/*
- * The MIT License (MIT)
- *
- * Copyright 2013 Andrey Sitnik <andrey@sitnik.ru>
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy of
- * this software and associated documentation files (the "Software"), to deal in
- * the Software without restriction, including without limitation the rights to
- * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
- * the Software, and to permit persons to whom the Software is furnished to do so,
- * subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in all
- * copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
- * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
- * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
- * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
- */
-
-var SINGLE_QUOTE = '\''.charCodeAt(0);
-var DOUBLE_QUOTE = '"'.charCodeAt(0);
-var BACKSLASH = '\\'.charCodeAt(0);
-var SLASH = '/'.charCodeAt(0);
-var NEWLINE = '\n'.charCodeAt(0);
-var SPACE = ' '.charCodeAt(0);
-var FEED = '\f'.charCodeAt(0);
-var TAB = '\t'.charCodeAt(0);
-var CR = '\r'.charCodeAt(0);
-var OPEN_SQUARE = '['.charCodeAt(0);
-var CLOSE_SQUARE = ']'.charCodeAt(0);
-var OPEN_PARENTHESES = '('.charCodeAt(0);
-var CLOSE_PARENTHESES = ')'.charCodeAt(0);
-var OPEN_CURLY = '{'.charCodeAt(0);
-var CLOSE_CURLY = '}'.charCodeAt(0);
-var SEMICOLON = ';'.charCodeAt(0);
-var ASTERISK = '*'.charCodeAt(0);
-var COLON = ':'.charCodeAt(0);
-var AT = '@'.charCodeAt(0);
-
-var RE_AT_END = /[ \n\t\r\f\{\(\)'"\\;/\[\]#]/g;
-var RE_WORD_END = /[ \n\t\r\f\(\)\{\}:;@!'"\\\]\[#]|\/(?=\*)/g;
-var RE_BAD_BRACKET = /.[\\\/\("'\n]/;
-var RE_HEX_ESCAPE = /[a-f0-9]/i;
-
-function tokenize(css) {
- var input = {css: css, error: function() {}};
- var processor = tokenizer(input);
- var tokens = [];
- while (!processor.endOfFile()) {
- tokens.push(processor.nextToken());
- }
- return tokens;
-}
-
-function tokenizer(input ) {
- var options = {};
- var css = input.css.valueOf();
- var ignore = options.ignoreErrors;
-
- var code, next, quote, lines, last, content, escape,
- nextLine, nextOffset, escaped, escapePos, prev, n, currentToken;
-
- var length = css.length;
- var offset = -1;
- var line = 1;
- var pos = 0;
- var buffer = [];
- var returned = [];
-
- function unclosed(what) {
- throw input.error('Unclosed ' + what, line, pos - offset);
- }
-
- function endOfFile() {
- return returned.length === 0 && pos >= length;
- }
-
- function nextToken() {
- if ( returned.length ) return returned.pop();
- if ( pos >= length ) return;
-
- code = css.charCodeAt(pos);
- if ( code === NEWLINE || code === FEED ||
- code === CR && css.charCodeAt(pos + 1) !== NEWLINE ) {
- offset = pos;
- line += 1;
- }
-
- switch ( code ) {
- case NEWLINE:
- case SPACE:
- case TAB:
- case CR:
- case FEED:
- next = pos;
- do {
- next += 1;
- code = css.charCodeAt(next);
- if ( code === NEWLINE ) {
- offset = next;
- line += 1;
- }
- } while ( code === SPACE ||
- code === NEWLINE ||
- code === TAB ||
- code === CR ||
- code === FEED );
-
- currentToken = ['space', css.slice(pos, next)];
- pos = next - 1;
- break;
-
- case OPEN_SQUARE:
- currentToken = ['[', '[', line, pos - offset];
- break;
-
- case CLOSE_SQUARE:
- currentToken = [']', ']', line, pos - offset];
- break;
-
- case OPEN_CURLY:
- currentToken = ['{', '{', line, pos - offset];
- break;
-
- case CLOSE_CURLY:
- currentToken = ['}', '}', line, pos - offset];
- break;
-
- case COLON:
- currentToken = [':', ':', line, pos - offset];
- break;
-
- case SEMICOLON:
- currentToken = [';', ';', line, pos - offset];
- break;
-
- case OPEN_PARENTHESES:
- prev = buffer.length ? buffer.pop()[1] : '';
- n = css.charCodeAt(pos + 1);
- if ( prev === 'url' &&
- n !== SINGLE_QUOTE && n !== DOUBLE_QUOTE &&
- n !== SPACE && n !== NEWLINE && n !== TAB &&
- n !== FEED && n !== CR ) {
- next = pos;
- do {
- escaped = false;
- next = css.indexOf(')', next + 1);
- if ( next === -1 ) {
- if ( ignore ) {
- next = pos;
- break;
- } else {
- unclosed('bracket');
- }
- }
- escapePos = next;
- while ( css.charCodeAt(escapePos - 1) === BACKSLASH ) {
- escapePos -= 1;
- escaped = !escaped;
- }
- } while ( escaped );
-
- currentToken = ['brackets', css.slice(pos, next + 1),
- line, pos - offset,
- line, next - offset
- ];
-
- pos = next;
-
- } else {
- next = css.indexOf(')', pos + 1);
- content = css.slice(pos, next + 1);
-
- if ( next === -1 || RE_BAD_BRACKET.test(content) ) {
- currentToken = ['(', '(', line, pos - offset];
- } else {
- currentToken = ['brackets', content,
- line, pos - offset,
- line, next - offset
- ];
- pos = next;
- }
- }
-
- break;
-
- case CLOSE_PARENTHESES:
- currentToken = [')', ')', line, pos - offset];
- break;
-
- case SINGLE_QUOTE:
- case DOUBLE_QUOTE:
- quote = code === SINGLE_QUOTE ? '\'' : '"';
- next = pos;
- do {
- escaped = false;
- next = css.indexOf(quote, next + 1);
- if ( next === -1 ) {
- if ( ignore ) {
- next = pos + 1;
- break;
- } else {
- unclosed('string');
- }
- }
- escapePos = next;
- while ( css.charCodeAt(escapePos - 1) === BACKSLASH ) {
- escapePos -= 1;
- escaped = !escaped;
- }
- } while ( escaped );
-
- content = css.slice(pos, next + 1);
- lines = content.split('\n');
- last = lines.length - 1;
-
- if ( last > 0 ) {
- nextLine = line + last;
- nextOffset = next - lines[last].length;
- } else {
- nextLine = line;
- nextOffset = offset;
- }
-
- currentToken = ['string', css.slice(pos, next + 1),
- line, pos - offset,
- nextLine, next - nextOffset
- ];
-
- offset = nextOffset;
- line = nextLine;
- pos = next;
- break;
-
- case AT:
- RE_AT_END.lastIndex = pos + 1;
- RE_AT_END.test(css);
- if ( RE_AT_END.lastIndex === 0 ) {
- next = css.length - 1;
- } else {
- next = RE_AT_END.lastIndex - 2;
- }
-
- currentToken = ['at-word', css.slice(pos, next + 1),
- line, pos - offset,
- line, next - offset
- ];
-
- pos = next;
- break;
-
- case BACKSLASH:
- next = pos;
- escape = true;
- while ( css.charCodeAt(next + 1) === BACKSLASH ) {
- next += 1;
- escape = !escape;
- }
- code = css.charCodeAt(next + 1);
- if ( escape && (code !== SLASH &&
- code !== SPACE &&
- code !== NEWLINE &&
- code !== TAB &&
- code !== CR &&
- code !== FEED ) ) {
- next += 1;
- if ( RE_HEX_ESCAPE.test(css.charAt(next)) ) {
- while ( RE_HEX_ESCAPE.test(css.charAt(next + 1)) ) {
- next += 1;
- }
- if ( css.charCodeAt(next + 1) === SPACE ) {
- next += 1;
- }
- }
- }
-
- currentToken = ['word', css.slice(pos, next + 1),
- line, pos - offset,
- line, next - offset
- ];
-
- pos = next;
- break;
-
- default:
- if ( code === SLASH && css.charCodeAt(pos + 1) === ASTERISK ) {
- next = css.indexOf('*/', pos + 2) + 1;
- if ( next === 0 ) {
- if ( ignore ) {
- next = css.length;
- } else {
- unclosed('comment');
- }
- }
-
- content = css.slice(pos, next + 1);
- lines = content.split('\n');
- last = lines.length - 1;
-
- if ( last > 0 ) {
- nextLine = line + last;
- nextOffset = next - lines[last].length;
- } else {
- nextLine = line;
- nextOffset = offset;
- }
-
- currentToken = ['comment', content,
- line, pos - offset,
- nextLine, next - nextOffset
- ];
-
- offset = nextOffset;
- line = nextLine;
- pos = next;
-
- } else {
- RE_WORD_END.lastIndex = pos + 1;
- RE_WORD_END.test(css);
- if ( RE_WORD_END.lastIndex === 0 ) {
- next = css.length - 1;
- } else {
- next = RE_WORD_END.lastIndex - 2;
- }
-
- currentToken = ['word', css.slice(pos, next + 1),
- line, pos - offset,
- line, next - offset
- ];
-
- buffer.push(currentToken);
-
- pos = next;
- }
-
- break;
- }
-
- pos++;
- return currentToken;
- }
-
- function back(token) {
- returned.push(token);
- }
-
- return {
- back:back, nextToken:nextToken, endOfFile:endOfFile
- };
-}
diff --git a/sonar-css-plugin/src/test/java/org/sonar/css/plugin/MetricSensorTest.java b/sonar-css-plugin/src/test/java/org/sonar/css/plugin/MetricSensorTest.java
index 6af504b..e8eb31b 100644
--- a/sonar-css-plugin/src/test/java/org/sonar/css/plugin/MetricSensorTest.java
+++ b/sonar-css-plugin/src/test/java/org/sonar/css/plugin/MetricSensorTest.java
@@ -71,7 +71,7 @@ public class MetricSensorTest {
highlight("\"foo\"");
assertHighlighting(1, 0, 5, TypeOfText.STRING);
- highlight("\"foo\nbar\"");
+ highlight("\"foo\\\nbar\"");
assertHighlighting(1, 0, 4, TypeOfText.STRING);
assertHighlighting(2, 0, 4, TypeOfText.STRING);
}
@@ -111,22 +111,11 @@ public class MetricSensorTest {
@Test
public void keyword() throws IOException {
- highlight("foo { }");
- assertHighlighting(1, 0, 3, TypeOfText.KEYWORD);
-
- highlight(".foo { }");
- assertHighlighting(1, 0, 4, TypeOfText.KEYWORD);
-
- highlight(".foo bar { }");
+ highlight("$foo { }");
assertHighlighting(1, 0, 4, TypeOfText.KEYWORD);
- assertHighlighting(1, 5, 3, TypeOfText.KEYWORD);
-
- highlight(".border-radius(@radius) { }");
- assertHighlighting(1, 0, 14, TypeOfText.KEYWORD);
highlight("#header { .border-radius(4px); }");
assertHighlighting(1, 0, 7, TypeOfText.KEYWORD);
- assertHighlighting(1, 10, 14, TypeOfText.KEYWORD);
}
@Test
diff --git a/sonar-css-plugin/src/test/java/org/sonar/css/plugin/TokenizerTest.java b/sonar-css-plugin/src/test/java/org/sonar/css/plugin/TokenizerTest.java
index b1589c0..76c4617 100644
--- a/sonar-css-plugin/src/test/java/org/sonar/css/plugin/TokenizerTest.java
+++ b/sonar-css-plugin/src/test/java/org/sonar/css/plugin/TokenizerTest.java
@@ -20,11 +20,7 @@
package org.sonar.css.plugin;
import java.util.List;
-import java.util.Optional;
-
-import javax.script.ScriptException;
import org.junit.Test;
-import org.sonar.css.plugin.Token.Type;
import static org.assertj.core.api.Assertions.assertThat;
@@ -33,67 +29,84 @@ public class TokenizerTest {
private final static Tokenizer tokenizer = new Tokenizer();
@Test
- public void word() throws ScriptException {
- assertToken("bar { }", 0, "bar", Type.WORD);
- assertToken("bar: foo { }", 0, "bar", Type.WORD);
- assertToken("bar: foo-baz { }", 2, "foo-baz", Type.WORD);
- assertToken("foo bar { }", 1, "bar", Type.WORD);
- assertToken("#bar { }", 0, "#bar", Type.WORD);
- assertToken("foo.bar { }", 0, "foo.bar", Type.WORD);
- assertToken(".bar { }", 0, ".bar", Type.WORD);
- assertToken("bar { foo: 42; }", 2, "foo", Type.WORD);
- assertToken("bar { foo: baz; }", 4, "baz", Type.WORD);
- assertToken("foo , bar { }", 2, "bar", Type.WORD);
+ public void identifier() {
+ assertToken("bar { }", 0, "bar", CssTokenType.IDENTIFIER);
+ assertToken("bar: foo { }", 0, "bar", CssTokenType.IDENTIFIER);
+ assertToken("bar: foo-baz { }", 2, "foo-baz", CssTokenType.IDENTIFIER);
+ assertToken("foo bar { }", 1, "bar", CssTokenType.IDENTIFIER);
+ assertToken("foo.bar { }", 0, "foo", CssTokenType.IDENTIFIER);
+ assertToken(".bar { }", 1, "bar", CssTokenType.IDENTIFIER);
+ assertToken("bar { foo: 42; }", 2, "foo", CssTokenType.IDENTIFIER);
+ assertToken("bar { foo: baz; }", 4, "baz", CssTokenType.IDENTIFIER);
+ assertToken("foo , bar { }", 2, "bar", CssTokenType.IDENTIFIER);
+
+ // support unicode characters
+ assertToken("\u03A9 { }", 0, "\u03A9", CssTokenType.IDENTIFIER);
+ }
+
+ @Test
+ public void at_identifier() {
+ assertToken("@bar { }", 0, "@bar", CssTokenType.AT_IDENTIFIER);
+ }
+
+ @Test
+ public void hash_identifier() {
+ assertToken("#bar { }", 0, "#bar", CssTokenType.HASH_IDENTIFIER);
+ assertToken("bar { color: #333; }", 4, "#333", CssTokenType.HASH_IDENTIFIER);
+ assertToken("bar { color: #e535ab; }", 4, "#e535ab", CssTokenType.HASH_IDENTIFIER);
}
@Test
- public void semi_colon() throws ScriptException {
- assertToken("bar: foo { }", 1, ":", Type.PUNCTUATOR);
- assertToken("bar { foo; }", 3, ";", Type.PUNCTUATOR);
+ public void semi_colon() {
+ assertToken("bar { foo; }", 3, ";", CssTokenType.PUNCTUATOR);
}
@Test
- public void comma() throws ScriptException {
- assertToken("foo , bar { }", 1, ",", Type.PUNCTUATOR);
- assertToken("foo, bar { }", 1, ",", Type.PUNCTUATOR);
+ public void colon() {
+ assertToken("bar { foo: 2px; }", 3, ":", CssTokenType.PUNCTUATOR);
}
@Test
- public void number_as_word() throws ScriptException {
- assertToken("bar { foo: 1.15; }", 4, "1.15", Type.WORD);
- assertToken("bar { foo: 1; }", 4, "1", Type.WORD);
- assertToken("bar { foo: 1.15px; }", 4, "1.15px", Type.WORD);
- assertToken("bar { foo: 1.15%; }", 4, "1.15%", Type.WORD);
- assertToken("bar { foo: 1px; }", 4, "1px", Type.WORD);
- assertToken("bar { foo: 1em/150%; }", 4, "1em/150%", Type.WORD);
+ public void comma() {
+ assertToken("foo , bar { }", 1, ",", CssTokenType.PUNCTUATOR);
+ assertToken("foo, bar { }", 1, ",", CssTokenType.PUNCTUATOR);
}
@Test
- public void brackets() throws ScriptException {
- assertToken("bar { foo: (1.15); }", 4, "(1.15)", Type.BRACKETS);
- assertToken("bar { foo: ( 1.15 ); }", 4, "( 1.15 )", Type.BRACKETS);
- assertToken("bar { foo: (1.15 1 0px); }", 4, "(1.15 1 0px)", Type.BRACKETS);
- assertToken("bar { foo: (1.15, 1, 0px); }", 4, "(1.15, 1, 0px)", Type.BRACKETS);
- assertToken("bar { content: string(doctitle); }", 5, "(doctitle)", Type.BRACKETS);
- assertToken("bar { string-set: booktitle content(); }", 6, "()", Type.BRACKETS);
- assertToken("bar { a: b(attr(href, url), c) \")\"; }", 7, "(href, url)", Type.BRACKETS);
+ public void number() {
+ assertToken("1.15", 0, "1.15", CssTokenType.NUMBER);
+ assertToken("1", 0, "1", CssTokenType.NUMBER);
+ assertToken("1.15px", 0, "1.15px", CssTokenType.NUMBER);
+ assertToken("1.15%", 0, "1.15%", CssTokenType.NUMBER);
+ assertToken("1px", 0, "1px", CssTokenType.NUMBER);
+ assertToken("1em/150%", 0, "1em", CssTokenType.NUMBER);
}
@Test
- public void strings() throws ScriptException {
- assertToken("bar { foo: \"\"; }", 4, "\"\"", Type.STRING);
- assertToken("bar { foo: \"hello, world\"; }", 4, "\"hello, world\"", Type.STRING);
+ public void parenthesis() {
+ assertToken("bar { foo: (1.15); }", 4, "(", CssTokenType.PUNCTUATOR);
+ assertToken("bar { foo: (1.15); }", 6, ")", CssTokenType.PUNCTUATOR);
+ assertToken("bar { foo: ( 1.15 ); }", 4, "(", CssTokenType.PUNCTUATOR);
+ assertToken("bar { foo: (1.15 1 0px); }", 4, "(", CssTokenType.PUNCTUATOR);
+ assertToken("bar { foo: (1.15, 1, 0px); }", 4, "(", CssTokenType.PUNCTUATOR);
+ assertToken("bar { content: string(doctitle); }", 5, "(", CssTokenType.PUNCTUATOR);
+ assertToken("bar { string-set: booktitle content(); }", 6, "(", CssTokenType.PUNCTUATOR);
+ assertToken("bar { a: b(attr(href, url), c) \")\"; }", 7, "(", CssTokenType.PUNCTUATOR);
}
@Test
- public void at_word() throws ScriptException {
- assertToken("@bar { }", 0, "@bar", Type.AT_WORD);
+ public void strings() {
+ assertToken("bar { foo: \"text\"; }", 4, "\"text\"", CssTokenType.STRING);
+ assertToken("bar { foo: \"hello, world\"; }", 4, "\"hello, world\"", CssTokenType.STRING);
+ assertToken("bar { foo: \"\"; }", 4, "\"\"", CssTokenType.STRING);
+ assertToken("\"foo\\\nbar\"", 0, "\"foo\\\nbar\"", CssTokenType.STRING);
+ assertToken("@min768: ~\"(min-width: 768px)\"", 2, "~\"(min-width: 768px)\"", CssTokenType.STRING);
}
@Test
- public void comment() throws ScriptException {
- assertToken("/* foo */", 0, "/* foo */", Type.COMMENT);
- assertToken("foo { a: /* foo */ 42; }", 4, "/* foo */", Type.COMMENT);
+ public void comment() {
+ assertToken("/* foo */", 0, "/* foo */", CssTokenType.COMMENT);
+ assertToken("foo { a: /* foo */ 42; }", 4, "/* foo */", CssTokenType.COMMENT);
assertToken("/* \n"
+ " this is a comment\n"
+ " and it is awesome because\n"
@@ -102,122 +115,110 @@ public class TokenizerTest {
+ " this is a comment\n"
+ " and it is awesome because\n"
+ " it is multiline!\n"
- + "*/", Type.COMMENT, 1, 1, 5, 2);
- assertToken("foo { a: /* foo\nbar*/ 42; }", 4, "/* foo\nbar*/", Type.COMMENT, 1, 10, 2, 5);
+ + "*/", CssTokenType.COMMENT, 1, 0, 5, 2);
+ assertToken("foo { a: /* foo\nbar*/ 42; }", 4, "/* foo\nbar*/", CssTokenType.COMMENT, 1, 9, 2, 5);
}
@Test
- public void hashtag() throws ScriptException {
- assertToken("bar { color: #333; }", 4, "#333", Type.WORD);
- assertToken("bar { color: #e535ab; }", 4, "#e535ab", Type.WORD);
- assertToken("#table-of-contents + ul li { list-style: none; }", 0, "#table-of-contents", Type.WORD);
- }
-
- @Test
- public void scss_variable() throws ScriptException {
- assertToken("$font-stack: Helvetica;", 0, "$font-stack", Type.WORD);
- assertToken("$message-color: blue !default;", 3, "!default", Type.WORD);
-
- List<Token> tokenList = tokenizer.tokenize("p.message-#{$alertClass} { color: red; }");
- assertThat(tokenList.size()).isEqualTo(11);
- assertToken(tokenList, 0, "p.message-", Type.WORD);
- assertToken(tokenList, 1, "#", Type.WORD);
- assertToken(tokenList, 2, "{", Type.PUNCTUATOR);
- assertToken(tokenList, 3, "$alertClass", Type.WORD);
- assertToken(tokenList, 4, "}", Type.PUNCTUATOR);
- assertToken(tokenList, 5, "{", Type.PUNCTUATOR);
- assertToken(tokenList, 6, "color", Type.WORD);
- assertToken(tokenList, 7, ":", Type.PUNCTUATOR);
- assertToken(tokenList, 8, "red", Type.WORD);
- assertToken(tokenList, 9, ";", Type.PUNCTUATOR);
- assertToken(tokenList, 10, "}", Type.PUNCTUATOR);
+ public void scss_variable() {
+ assertToken("$font-stack: Helvetica;", 0, "$font-stack", CssTokenType.DOLLAR_IDENTIFIER);
+ assertToken("$message-color: blue !default;", 4, "default", CssTokenType.IDENTIFIER);
+ List<CssToken> tokenList = tokenizer.tokenize("p.message-#{$alertClass} { color: red; }");
+ assertThat(tokenList.size()).isEqualTo(13);
+ assertToken(tokenList, 0, "p", CssTokenType.IDENTIFIER);
+ assertToken(tokenList, 1, ".", CssTokenType.PUNCTUATOR);
+ assertToken(tokenList, 2, "message-", CssTokenType.IDENTIFIER);
+ assertToken(tokenList, 3, "#", CssTokenType.PUNCTUATOR);
+ assertToken(tokenList, 4, "{", CssTokenType.PUNCTUATOR);
+ assertToken(tokenList, 5, "$alertClass", CssTokenType.DOLLAR_IDENTIFIER);
+ assertToken(tokenList, 6, "}", CssTokenType.PUNCTUATOR);
+ assertToken(tokenList, 7, "{", CssTokenType.PUNCTUATOR);
+ assertToken(tokenList, 8, "color", CssTokenType.IDENTIFIER);
+ assertToken(tokenList, 9, ":", CssTokenType.PUNCTUATOR);
+ assertToken(tokenList, 10, "red", CssTokenType.IDENTIFIER);
+ assertToken(tokenList, 11, ";", CssTokenType.PUNCTUATOR);
+ assertToken(tokenList, 12, "}", CssTokenType.PUNCTUATOR);
}
@Test
- public void scss_import() throws ScriptException {
- List<Token> tokenList = tokenizer.tokenize("@import 'base';");
+ public void scss_import() {
+ List<CssToken> tokenList = tokenizer.tokenize("@import 'base';");
assertThat(tokenList.size()).isEqualTo(3);
- assertToken(tokenList, 0, "@import", Type.AT_WORD);
- assertToken(tokenList, 1, "'base'", Type.STRING);
- assertToken(tokenList, 2, ";", Type.PUNCTUATOR);
+ assertToken(tokenList, 0, "@import", CssTokenType.AT_IDENTIFIER);
+ assertToken(tokenList, 1, "'base'", CssTokenType.STRING);
+ assertToken(tokenList, 2, ";", CssTokenType.PUNCTUATOR);
}
@Test
- public void scss_role() throws ScriptException {
- List<Token> tokenList = tokenizer.tokenize("article[role=\"main\"] { width: 1px; }");
+ public void scss_role() {
+ List<CssToken> tokenList = tokenizer.tokenize("article[role=\"main\"] { width: 1px; }");
- assertThat(tokenList.size()).isEqualTo(11);
- assertToken(tokenList, 0, "article", Type.WORD);
- assertToken(tokenList, 1, "[", Type.PUNCTUATOR);
- assertToken(tokenList, 2, "role=", Type.WORD);
- assertToken(tokenList, 3, "\"main\"", Type.STRING);
- assertToken(tokenList, 4, "]", Type.PUNCTUATOR);
+ assertThat(tokenList.size()).isEqualTo(12);
+ assertToken(tokenList, 0, "article", CssTokenType.IDENTIFIER);
+ assertToken(tokenList, 1, "[", CssTokenType.PUNCTUATOR);
+ assertToken(tokenList, 2, "role", CssTokenType.IDENTIFIER);
+ assertToken(tokenList, 3, "=", CssTokenType.PUNCTUATOR);
+ assertToken(tokenList, 4, "\"main\"", CssTokenType.STRING);
+ assertToken(tokenList, 5, "]", CssTokenType.PUNCTUATOR);
}
@Test
- public void scss_operators() throws ScriptException {
- assertToken("foo { width: 300px + 960px; }", 5, "+", Type.WORD);
- assertToken("foo { width: 300px - 960px; }", 5, "-", Type.WORD);
- assertToken("foo { width: 300px * 960px; }", 5, "*", Type.WORD);
- assertToken("foo { width: 300px / 960px; }", 5, "/", Type.WORD);
+ public void scss_less_operators() {
+ assertToken("foo { width: 300px + 960px; }", 5, "+", CssTokenType.PUNCTUATOR);
+ assertToken("foo { width: 300px - 960px; }", 5, "-", CssTokenType.PUNCTUATOR);
+ assertToken("foo { width: 300px * 960px; }", 5, "*", CssTokenType.PUNCTUATOR);
+ assertToken("foo { width: 300px / 960px; }", 5, "/", CssTokenType.PUNCTUATOR);
}
@Test
- public void scss_parent_selector() throws ScriptException {
- assertToken("a { &:hover { color: red; } }", 2, "&", Type.WORD);
- assertToken("p { body.no-touch & { display: none; } }", 3, "&", Type.WORD);
+ public void scss_parent_selector() {
+ assertToken("a { &:hover { color: red; } }", 2, "&", CssTokenType.PUNCTUATOR);
+ assertToken("p { body.no-touch & { display: none; } }", 5, "&", CssTokenType.PUNCTUATOR);
}
@Test
- public void scss_control_directives() throws ScriptException {
- assertToken("@if ($debug) { }", 0, "@if", Type.AT_WORD);
- assertToken("@each $name in 'save' 'cancel' { }", 0, "@each", Type.AT_WORD);
+ public void scss_control_directives() {
+ assertToken("@if ($debug) { }", 0, "@if", CssTokenType.AT_IDENTIFIER);
+ assertToken("@each $name in 'save' 'cancel' { }", 0, "@each", CssTokenType.AT_IDENTIFIER);
}
@Test
- public void less_variable() throws ScriptException {
- assertToken("@nice-blue: #5B83AD;", 0, "@nice-blue", Type.AT_WORD);
- assertToken("foo { color: @@color; }", 4, "@@color", Type.AT_WORD);
+ public void less_variable() {
+ assertToken("@nice-blue: #5B83AD;", 0, "@nice-blue", CssTokenType.AT_IDENTIFIER);
+ assertToken("foo { color: @@color; }", 4, "@@color", CssTokenType.AT_IDENTIFIER);
}
@Test
- public void less_operators() throws ScriptException {
- assertToken("@base: 2cm * 3mm;", 3, "*", Type.WORD);
+ public void less_comment() {
+ assertToken("// Get in line!", 0, "// Get in line!", CssTokenType.COMMENT);
+ assertToken("// body font size = 62.5%\n\n/* some comment */", 0, "// body font size = 62.5%", CssTokenType.COMMENT);
+ assertToken("/* One heck of a block\n * style comment! */", 0, "/* One heck of a block\n * style comment! */", CssTokenType.COMMENT);
}
@Test
- public void less_escaping() throws ScriptException {
- assertToken("@min768: ~\"(min-width: 768px)\";", 2, "~", Type.WORD);
- }
-
- @Test
- public void less_comment() throws ScriptException {
- // FIXME: Less allows // comment which are not supported by our current tokenizer
- //assertToken("// Get in line!", 0, "Get in line!", Type.COMMENT);
-
- assertToken("/* One heck of a block\n * style comment! */", 0, "/* One heck of a block\n * style comment! */", Type.COMMENT);
+ public void unrecognized() {
+ assertToken("$$a", 0, "$a", CssTokenType.DOLLAR_IDENTIFIER);
}
- private static void assertToken(String input, int index, String value, Token.Type type) throws ScriptException {
- List<Token> tokenList = tokenizer.tokenize(input);
- assertToken(tokenList, index, value, type);
+ private static void assertToken(String input, int index, String value, CssTokenType CssTokenType) {
+ List<CssToken> tokenList = tokenizer.tokenize(input);
+ assertToken(tokenList, index, value, CssTokenType);
}
- private static void assertToken(String input, int index, String value, Token.Type type, int line, int column, int
- endLine, int endColumn) throws ScriptException {
- List<Token> tokenList = tokenizer.tokenize(input);
- assertToken(tokenList, index, value, type, line, column, endLine, endColumn);
+ private static void assertToken(String input, int index, String value, CssTokenType CssTokenType, int line, int column, int endLine, int endColumn) {
+ List<CssToken> tokenList = tokenizer.tokenize(input);
+ assertToken(tokenList, index, value, CssTokenType, line, column, endLine, endColumn);
}
- private static void assertToken(List<Token> tokenList, int index, String value, Token.Type type) {
- assertThat(tokenList.get(index).type).isEqualTo(type);
+ private static void assertToken(List<CssToken> tokenList, int index, String value, CssTokenType CssTokenType) {
+ assertThat(tokenList.get(index).type).isEqualTo(CssTokenType);
assertThat(tokenList.get(index).text).isEqualTo(value);
}
- private static void assertToken(List<Token> tokenList, int index, String value, Token.Type type, int line, int column, int endLine, int endColumn) {
- assertToken(tokenList, index, value, type);
+ private static void assertToken(List<CssToken> tokenList, int index, String value, CssTokenType CssTokenType, int line, int column, int endLine, int endColumn) {
+ assertToken(tokenList, index, value, CssTokenType);
assertThat(tokenList.get(index).startLine).isEqualTo(line);
assertThat(tokenList.get(index).startColumn).isEqualTo(column);
assertThat(tokenList.get(index).endLine).isEqualTo(endLine);
generated by cgit v1.2.3 (git 2.25.1) at 2025-10-31 11:04:34 +0000