Semantic Tokens ext point

org.eclipse.jdt.text.tests/plugin.xml

@@ -47,4 +47,10 @@
 			<partition type="__java_string"/>
 		</javaCompletionProposalComputer>
 	</extension>
+ <extension
+       point="org.eclipse.jdt.ui.semanticTokens">
+    <provider
+          class="org.eclipse.jdt.text.tests.semantictokens.SampleSqlSemanticTokensProvider">
+    </provider>
+ </extension>
 </plugin>

org.eclipse.jdt.text.tests/src/org/eclipse/jdt/text/tests/JdtTextTestSuite.java

@@ -1,5 +1,5 @@
 /*******************************************************************************
- * Copyright (c) 2000, 2023 IBM Corporation and others.
+ * Copyright (c) 2000, 2024 IBM Corporation and others.
  *
  * This program and the accompanying materials
  * are made available under the terms of the Eclipse Public License 2.0
@@ -20,6 +20,7 @@
 import org.eclipse.jdt.text.tests.codemining.CodeMiningTriggerTest;
 import org.eclipse.jdt.text.tests.codemining.ParameterNamesCodeMiningTest;
 import org.eclipse.jdt.text.tests.contentassist.ContentAssistTestSuite;
+import org.eclipse.jdt.text.tests.semantictokens.SemanticTokensProviderTest;
 import org.eclipse.jdt.text.tests.spelling.SpellCheckEngineTestCase;
 import org.eclipse.jdt.text.tests.templates.TemplatesTestSuite;
 
@@ -52,6 +53,7 @@
 	BracketInserterTest.class,
 	SpellCheckEngineTestCase.class,
 	SemanticHighlightingTest.class,
+	SemanticTokensProviderTest.class,
 	AutoboxingSemanticHighlightingTest.class,
 	Java23SemanticHighlightingTest.class,
 	NewForLoopJavaContextTest.class,

org.eclipse.jdt.text.tests/src/org/eclipse/jdt/text/tests/semantictokens/SampleSqlSemanticTokensProvider.java

@@ -0,0 +1,137 @@
+/*******************************************************************************
+ * Copyright (c) 2024 Broadcom Inc. and others.
+ * This program and the accompanying materials are made
+ * available under the terms of the Eclipse Public License 2.0
+ * which is available at https://www.eclipse.org/legal/epl-2.0/
+ *
+ * SPDX-License-Identifier: EPL-2.0
+ *
+ * Contributors:
+ *    Alex Boyko (Broadcom Inc.) - Initial implementation
+ *******************************************************************************/
+package org.eclipse.jdt.text.tests.semantictokens;
+
+import java.text.NumberFormat;
+import java.text.ParseException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.eclipse.jdt.core.dom.ASTVisitor;
+import org.eclipse.jdt.core.dom.CompilationUnit;
+import org.eclipse.jdt.core.dom.Expression;
+import org.eclipse.jdt.core.dom.StringLiteral;
+import org.eclipse.jdt.core.dom.TextBlock;
+
+import org.eclipse.jdt.ui.text.java.ISemanticTokensProvider;
+
+/**
+ * Semantic tokens are computed for string literals or text blocks starting with "SQL:" prefix.
+ * <ul>
+ * <li>SELECT, WHERE, IN, FROM are KEYWORD</li>
+ * <li>*, &lt;, &gt;, ==, != etc are OPEARATOR</li>
+ * <li>Words starting from capital are CLASS</li>
+ * <li>Numbers are NUMBER</li>
+ * <li>all other lower case starting words are LOCAL_VARIABLE</li>
+ * </ul>
+ */
+public class SampleSqlSemanticTokensProvider implements ISemanticTokensProvider {
+
+	private static final String SQL_PREFIX = "SQL:";
+
+	@Override
+	public Collection<SemanticToken> computeSemanticTokens(CompilationUnit ast) {
+		List<SemanticToken> tokens = new ArrayList<>();
+		ast.accept(new ASTVisitor() {
+
+			@Override
+			public boolean visit(StringLiteral node) {
+				tokens.addAll(reconileEmbeddedExpression(node));
+				return super.visit(node);
+			}
+
+			@Override
+			public boolean visit(TextBlock node) {
+				tokens.addAll(reconileEmbeddedExpression(node));
+				return super.visit(node);
+			}
+
+		});
+		return tokens;
+	}
+
+	private List<SemanticToken> reconileEmbeddedExpression(Expression valueExp) {
+		String text = null;
+		int offset = 0;
+		if (valueExp instanceof StringLiteral sl && sl.getLiteralValue().startsWith(SQL_PREFIX)) {
+			text = sl.getEscapedValue();
+			int skip = 1 + SQL_PREFIX.length();
+			text = text.substring(skip, text.length() - 1);
+			offset = sl.getStartPosition() + skip; // +1 to skip over opening " and over "SQL:"
+		} else if (valueExp instanceof TextBlock tb && tb.getLiteralValue().startsWith(SQL_PREFIX)) {
+			text = tb.getEscapedValue();
+			int skip = 3 + SQL_PREFIX.length();
+			text = text.substring(skip, text.length() - 3);
+			offset = tb.getStartPosition() + skip; // +3 to skip over opening """ and over "SQL:"
+		}
+		return compute(text, offset);
+	}
+
+	private List<SemanticToken> compute(String text, int offset) {
+		if (text == null) {
+			return Collections.emptyList();
+		}
+		List<SemanticToken> tokens = new ArrayList<>();
+		Matcher matcher= Pattern.compile("[\\w*=><!]+").matcher(text);
+		while (matcher.find()) {
+			String token = matcher.group();
+			if (!token.isBlank()) {
+				int start = matcher.start();
+				int end = matcher.end();
+				tokens.add(new SemanticToken(start + offset, end - start, getTokenType(token)));
+			}
+		}
+		return tokens;
+	}
+
+	private TokenType getTokenType(String token) {
+		try {
+			NumberFormat.getInstance().parse(token);
+			return TokenType.NUMBER;
+		} catch (ParseException e) {
+			switch (token) {
+				case "SELECT":
+				case "WHERE":
+				case "WHEN":
+				case "ALL":
+				case "BY":
+				case "ORDER":
+				case "LIKE":
+				case "IN":
+				case "FROM":
+				case "NOT":
+					return TokenType.KEYWORD;
+				case "*":
+				case "(":
+				case "-":
+				case ">":
+				case "<":
+				case ">=":
+				case "<=":
+				case "==":
+				case "!=":
+					return TokenType.OPERATOR;
+				default:
+					if (token.length() > 0 && Character.isUpperCase(token.charAt(0))) {
+						return TokenType.CLASS;
+					}
+					return TokenType.LOCAL_VARIABLE;
+			}
+		}
+	}
+
+
+}

org.eclipse.jdt.text.tests/src/org/eclipse/jdt/text/tests/semantictokens/SemanticTokensProviderTest.java

@@ -0,0 +1,50 @@
+/*******************************************************************************
+ * Copyright (c) 2024 Broadcom Inc. and others.
+ * This program and the accompanying materials are made
+ * available under the terms of the Eclipse Public License 2.0
+ * which is available at https://www.eclipse.org/legal/epl-2.0/
+ *
+ * SPDX-License-Identifier: EPL-2.0
+ *
+ * Contributors:
+ *    Alex Boyko (Broadcom Inc.) - Initial implementation
+ *******************************************************************************/
+package org.eclipse.jdt.text.tests.semantictokens;
+
+import org.junit.Rule;
+import org.junit.Test;
+
+import org.eclipse.jdt.text.tests.AbstractSemanticHighlightingTest;
+
+import org.eclipse.jface.text.Position;
+
+import org.eclipse.jdt.internal.ui.javaeditor.SemanticHighlightings;
+
+public class SemanticTokensProviderTest extends AbstractSemanticHighlightingTest {
+
+	@Rule
+	public SemanticHighlightingTestSetup shts= new SemanticHighlightingTestSetup( "/SHTest/src/STTest.java");
+
+	@Test
+	public void contributedHighlighting() throws Exception {
+		setUpSemanticHighlighting(SemanticHighlightings.CLASS);
+		setUpSemanticHighlighting(SemanticHighlightings.NUMBER);
+		setUpSemanticHighlighting(SemanticHighlightings.LOCAL_VARIABLE);
+		Position[] actual= getSemanticHighlightingPositions();
+		Position[] expected= new Position[] {
+				createPosition(0, 6, 1),
+				createPosition(1, 1, 6),
+				createPosition(1, 20, 6),
+				createPosition(1, 27, 1),
+				createPosition(1, 29, 4),
+				createPosition(1, 34, 1),
+				createPosition(1, 36, 5),
+				createPosition(1, 42, 1),
+				createPosition(1, 44, 2),
+				createPosition(1, 47, 3),
+		};
+		assertEqualPositions(expected, actual);
+	}
+
+
+}

org.eclipse.jdt.text.tests/testResources/semanticHighlightingTest1/STTest.java

@@ -0,0 +1,3 @@
+class A {
+	String sql = "SQL: SELECT * FROM T WHERE a == 567";
+}

org.eclipse.jdt.ui/META-INF/MANIFEST.MF

@@ -3,7 +3,7 @@ Automatic-Module-Name: org.eclipse.jdt.ui
 Bundle-ManifestVersion: 2
 Bundle-Name: %pluginName
 Bundle-SymbolicName: org.eclipse.jdt.ui; singleton:=true
-Bundle-Version: 3.33.300.qualifier
+Bundle-Version: 3.34.0.qualifier
 Bundle-Activator: org.eclipse.jdt.internal.ui.JavaPlugin
 Bundle-ActivationPolicy: lazy
 Bundle-Vendor: %providerName

org.eclipse.jdt.ui/plugin.properties

@@ -26,6 +26,7 @@ JavaSelectRulerAction.label= Java Editor Ruler Single-Click
 
 importExportWizards.category=Java
 elementFiltersName=Java Element Filters
+semanticTokens=Java Editor Semantic Tokens for Syntax Highlighting
 
 classpathContainerPageExtensionPoint=Classpath Container Configuration
 classpathAttributeConfiguration=Classpath Attribute Configuration
@@ -38,6 +39,8 @@ javadocExportWizardPageExtensionPoint=Javadoc Export Wizard Page
 
 cleanUpExtensionPoint=Clean Ups
 
+semanticTokensExtensionPoint=Semantic Tokens
+
 queryParticipantsExtensionPoint=Java Query Participants
 
 defaultClasspathContainerPage=Default Classpath Container

org.eclipse.jdt.ui/plugin.xml

@@ -34,6 +34,7 @@
    <extension-point id="classpathAttributeConfiguration" name="%classpathAttributeConfiguration" schema="schema/classpathAttributeConfiguration.exsd"/>
    <extension-point id="javadocExportWizardPage" name="%javadocExportWizardPageExtensionPoint" schema="schema/javadocExportWizardPage.exsd"/>
    <extension-point id="cleanUps" name="%cleanUpExtensionPoint" schema="schema/cleanUps.exsd"/>
+   <extension-point id="semanticTokens" name="%semanticTokensExtensionPoint" schema="schema/semanticTokens.exsd"/>
 
 <extension
 	point="org.eclipse.ui.decorators">

org.eclipse.jdt.ui/pom.xml

@@ -18,7 +18,7 @@
   </parent>
   <groupId>org.eclipse.jdt</groupId>
   <artifactId>org.eclipse.jdt.ui</artifactId>
-  <version>3.33.300-SNAPSHOT</version>
+  <version>3.34.0-SNAPSHOT</version>
   <packaging>eclipse-plugin</packaging>
 
 	<build>