Merge pull request #33104 from vespa-engine/magnus/langserver-windows

Magnus/langserver windows

integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/SchemaLanguageServer.java

@@ -86,7 +86,12 @@ public SchemaLanguageServer() {
 
         this.textDocumentService = new SchemaTextDocumentService(this.logger, schemaDocumentScheduler, schemaIndex, schemaMessageHandler);
         this.workspaceService = new SchemaWorkspaceService(this.logger, schemaDocumentScheduler, schemaIndex, schemaMessageHandler);
-        serverPath = Paths.get(SchemaLanguageServer.class.getProtectionDomain().getCodeSource().getLocation().getPath()).getParent();
+        try {
+            serverPath = Paths.get(new File(SchemaLanguageServer.class.getProtectionDomain().getCodeSource().getLocation().toURI()).getCanonicalPath()).getParent();
+        } catch(Exception ex) {
+            // This is the old way which crashed on Windows, but works on Unix systems.
+            serverPath = Paths.get(SchemaLanguageServer.class.getProtectionDomain().getCodeSource().getLocation().getPath()).getParent();
+        }
     }    
 
     @Override

integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/SchemaTextDocumentService.java

@@ -2,6 +2,8 @@
 
 import java.io.ByteArrayOutputStream;
 import java.io.PrintStream;
+import java.net.URLDecoder;
+import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.concurrent.CancellationException;
@@ -54,6 +56,7 @@
 import org.eclipse.lsp4j.services.TextDocumentService;
 
 import ai.vespa.schemals.common.ClientLogger;
+import ai.vespa.schemals.common.FileUtils;
 import ai.vespa.schemals.context.EventContextCreator;
 import ai.vespa.schemals.context.EventDocumentContext;
 import ai.vespa.schemals.context.InvalidContextException;
@@ -250,27 +253,29 @@ public void didChange(DidChangeTextDocumentParams params) {
 
         var contentChanges = params.getContentChanges();
         for (int i = 0; i < contentChanges.size(); i++) {
+            String fileURI = FileUtils.decodeURL(document.getUri());
             try {
-                scheduler.updateFile(document.getUri(), contentChanges.get(i).getText());
+                scheduler.updateFile(fileURI, contentChanges.get(i).getText());
             } catch(Exception e) {
                 ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
                 PrintStream logger = new PrintStream(outputStream);
 
                 e.printStackTrace(logger);
 
                 schemaMessageHandler.logMessage(MessageType.Error, 
-                    "Updating file " + document.getUri() + " failed with error: " + outputStream.toString() 
+                    "Updating file " + fileURI + " failed with error: " + outputStream.toString() 
                 );
             }
         }
     }
 
+
     @Override
     public void didClose(DidCloseTextDocumentParams params) {
         TextDocumentIdentifier documentIdentifier = params.getTextDocument();
         SchemaDocumentScheduler scheduler = eventContextCreator.scheduler;
 
-        scheduler.closeDocument(documentIdentifier.getUri());
+        scheduler.closeDocument(FileUtils.decodeURL(documentIdentifier.getUri()));
     }
 
     @Override

integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/SchemaWorkspaceService.java

@@ -14,6 +14,7 @@
 import org.eclipse.lsp4j.services.WorkspaceService;
 
 import ai.vespa.schemals.common.ClientLogger;
+import ai.vespa.schemals.common.FileUtils;
 import ai.vespa.schemals.context.EventContextCreator;
 import ai.vespa.schemals.index.SchemaIndex;
 import ai.vespa.schemals.lsp.common.command.ExecuteCommand;
@@ -44,7 +45,7 @@ public void didDeleteFiles(DeleteFilesParams params) {
         List<FileDelete> deletedFiles = params.getFiles();
 
         for (FileDelete file : deletedFiles) {
-            scheduler.removeDocument(file.getUri());
+            scheduler.removeDocument(FileUtils.decodeURL(file.getUri()));
         }
 
     }

integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/common/FileUtils.java

@@ -4,6 +4,8 @@
 import java.io.FileReader;
 import java.io.IOException;
 import java.net.URI;
+import java.net.URLDecoder;
+import java.nio.charset.StandardCharsets;
 import java.nio.file.FileSystems;
 import java.nio.file.FileVisitResult;
 import java.nio.file.Files;
@@ -15,6 +17,8 @@
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Optional;
+import java.util.Set;
+import java.util.stream.Collectors;
 
 import com.yahoo.io.IOUtils;
 
@@ -71,6 +75,26 @@ public static String firstPathComponentAfterPrefix(String pathURIStr, String pre
         return components[0];
     }
 
+    // https://stackoverflow.com/questions/1976007/what-characters-are-forbidden-in-windows-and-linux-directory-names
+    private final static Set<Character> DISALLOWED_CHARS = Set.of('/', '<', '>', ':', '"', '\\', '|', '?', '*');
+    public static String sanitizeFileName(String fileName) {
+        return fileName.chars()
+                       .filter(c -> !DISALLOWED_CHARS.contains((char) c))
+                       .mapToObj(c -> "" + (char) c)
+                       .collect(Collectors.joining());
+    }
+
+    /* 
+     * Decode URL in a kind way
+     */
+    public static String decodeURL(String URL) {
+        try {
+            return URLDecoder.decode(URL, StandardCharsets.UTF_8.name());
+        } catch(Exception e) {
+            return URL;
+        }
+    }
+
     /**
      * Searches among the parents for a directory named "schemas"
      */
@@ -99,7 +123,7 @@ private static List<String> walkFileTree(Path rootDir, String pathMatcherStr, Cl
                 @Override
                 public FileVisitResult visitFile(Path path, BasicFileAttributes attrs) throws IOException {
                     if (pathMatcher.matches(path)) {
-                        filePaths.add(path.toUri().toString());
+                        filePaths.add(decodeURL(path.toUri().toString()));
                     }
                     return FileVisitResult.CONTINUE;
                 }

integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/common/editbuilder/TextDocumentEditBuilder.java

@@ -6,6 +6,8 @@
 import org.eclipse.lsp4j.TextEdit;
 import org.eclipse.lsp4j.VersionedTextDocumentIdentifier;
 
+import ai.vespa.schemals.common.FileUtils;
+
 public class TextDocumentEditBuilder {
     private ArrayList<TextEdit> textEdits;
     private VersionedTextDocumentIdentifier versionedTextDocumentIdentifier;
@@ -31,7 +33,7 @@ public TextDocumentEditBuilder addEdit(TextEdit textEdit) {
     }
 
     public String getFileURI() {
-        return versionedTextDocumentIdentifier.getUri();
+        return FileUtils.decodeURL(versionedTextDocumentIdentifier.getUri());
     }
 
     public TextDocumentEdit build() {

integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/common/editbuilder/WorkspaceEditBuilder.java

@@ -12,6 +12,8 @@
 import org.eclipse.lsp4j.WorkspaceEdit;
 import org.eclipse.lsp4j.jsonrpc.messages.Either;
 
+import ai.vespa.schemals.common.FileUtils;
+
 
 /**
  * WorkspaceEditBuilder
@@ -34,18 +36,20 @@ public WorkspaceEditBuilder addTextEdit(String fileURI, TextEdit edit) {
     }
 
     public WorkspaceEditBuilder addTextEdit(VersionedTextDocumentIdentifier identifier, TextEdit edit) {
-        if (!textDocumentBuilders.containsKey(identifier.getUri())) {
+        String fileURI = FileUtils.decodeURL(identifier.getUri());
+        if (!textDocumentBuilders.containsKey(fileURI)) {
             registerVersionedDocumentIdentifier(identifier);
         }
-        textDocumentBuilders.get(identifier.getUri()).addEdit(edit);
+        textDocumentBuilders.get(fileURI).addEdit(edit);
         return this;
     }
 
     public WorkspaceEditBuilder addTextEdits(VersionedTextDocumentIdentifier identifier, Iterable<TextEdit> edits) {
-        if (!textDocumentBuilders.containsKey(identifier.getUri())) {
+        String fileURI = FileUtils.decodeURL(identifier.getUri());
+        if (!textDocumentBuilders.containsKey(fileURI)) {
             registerVersionedDocumentIdentifier(identifier);
         }
-        TextDocumentEditBuilder builder = textDocumentBuilders.get(identifier.getUri());
+        TextDocumentEditBuilder builder = textDocumentBuilders.get(fileURI);
         edits.forEach(edit -> builder.addEdit(edit));
         return this;
     }
@@ -57,7 +61,7 @@ public WorkspaceEditBuilder addResourceOperation(ResourceOperation resourceOpera
 
     public WorkspaceEditBuilder registerVersionedDocumentIdentifier(VersionedTextDocumentIdentifier identifier) {
         var builder = new TextDocumentEditBuilder().setVersionedTextDocumentIdentifier(identifier);
-        textDocumentBuilders.put(identifier.getUri(), builder);
+        textDocumentBuilders.put(FileUtils.decodeURL(identifier.getUri()), builder);
         edits.add(Either.forLeft(builder));
         return this;
     }

integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/context/EventDocumentContext.java

@@ -7,6 +7,7 @@
 import ai.vespa.schemals.schemadocument.DocumentManager;
 
 import ai.vespa.schemals.SchemaMessageHandler;
+import ai.vespa.schemals.common.FileUtils;
 import ai.vespa.schemals.index.SchemaIndex;
 import ai.vespa.schemals.schemadocument.SchemaDocumentScheduler;
 
@@ -22,7 +23,7 @@ public EventDocumentContext(
     ) throws InvalidContextException {
         super(scheduler, schemaIndex, messageHandler);
         this.documentIdentifier = documentIdentifier;
-        this.document = scheduler.getDocument(documentIdentifier.getUri());
+        this.document = scheduler.getDocument(FileUtils.decodeURL(documentIdentifier.getUri()));
         if (this.document == null) {
             throw new InvalidContextException();
         }

integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/documentation/FetchDocumentation.java

@@ -8,6 +8,10 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import ai.vespa.schemals.common.FileUtils;
 
 /**
  * DocumentationFetcher
@@ -54,6 +58,7 @@ public static void fetchSchemaDocs(Path targetPath) throws IOException {
             List<String> fileNamesToWrite = REPLACE_FILENAME_MAP.getOrDefault(tokenName, List.of(tokenName));
 
             for (String fileName : fileNamesToWrite) {
+                fileName = FileUtils.sanitizeFileName(fileName);
                 writeMarkdown(writePath.resolve(fileName + ".md"), content);
             }
         }
@@ -62,7 +67,8 @@ public static void fetchSchemaDocs(Path targetPath) throws IOException {
 
         writePath = targetPath.resolve("rankExpression");
         for (var entry : rankFeatureMarkdownContent.entrySet()) {
-            writeMarkdown(writePath.resolve(entry.getKey() + ".md"), entry.getValue());
+            String fileName = FileUtils.sanitizeFileName(entry.getKey());
+            writeMarkdown(writePath.resolve(fileName + ".md"), entry.getValue());
         }
     }
 
@@ -79,11 +85,13 @@ public static void fetchServicesDocs(Path targetPath) throws IOException {
             for (var entry : markdownContent.entrySet()) {
                 if (entry.getKey().contains("/")) continue;
                 String fileName = entry.getKey().toLowerCase();
+                fileName = FileUtils.sanitizeFileName(fileName);
                 writeMarkdown(writePath.resolve(fileName + ".md"), entry.getValue());
             }
         }
     }
 
+
     private static void writeMarkdown(Path writePath, String markdown) throws IOException {
         Files.write(writePath, markdown.getBytes(), StandardOpenOption.CREATE);
     }

integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/lsp/schema/completion/provider/BodyKeywordCompletion.java

@@ -181,8 +181,8 @@ public class BodyKeywordCompletion implements CompletionProvider {
         // compute docs
         for (var entry : this.entrySet()) {
             for (CompletionItem item : entry.getValue()) {
-                String markdownKey = "schema/" + item.getLabel().toUpperCase().replaceAll("-", "_");
-                Optional<Hover> hover = SchemaHover.getFileHoverInformation(markdownKey, new Range());
+                String markdownKey = item.getLabel().toUpperCase().replaceAll("-", "_");
+                Optional<Hover> hover = SchemaHover.getFileHoverInformation("schema", markdownKey, new Range());
                 if (hover.isPresent() && hover.get().getContents().isRight()) {
                     item.setDocumentation(hover.get().getContents().getRight());
                 }

integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/lsp/schema/hover/SchemaHover.java

@@ -19,6 +19,7 @@
 import org.eclipse.lsp4j.Range;
 
 import ai.vespa.schemals.SchemaLanguageServer;
+import ai.vespa.schemals.common.FileUtils;
 import ai.vespa.schemals.context.EventPositionContext;
 import ai.vespa.schemals.index.Symbol;
 import ai.vespa.schemals.index.Symbol.SymbolStatus;
@@ -288,13 +289,13 @@ private static Optional<Hover> rankFeatureHover(SchemaNode node, EventPositionCo
      * @return Hover with empty range
      */
     public static Optional<Hover> getRankFeatureHover(SpecificFunction function) {
-        Optional<Hover> result = getFileHoverInformation("rankExpression/" + function.getSignatureString(), new Range());
+        Optional<Hover> result = getFileHoverInformation("rankExpression", function.getSignatureString(), new Range());
 
         if (result.isPresent()) {
             return result;
         }
 
-        return getFileHoverInformation("rankExpression/" + function.getSignatureString(true), new Range());
+        return getFileHoverInformation("rankExpression", function.getSignatureString(true), new Range());
     }
 
     public static Hover getHover(EventPositionContext context) {
@@ -329,14 +330,14 @@ public static Hover getHover(EventPositionContext context) {
             return getIndexingHover(schemaNode, context);
         }
 
-        Optional<Hover> hoverInfo = getFileHoverInformation("schema/" + schemaNode.getClassLeafIdentifierString(), schemaNode.getRange());
+        Optional<Hover> hoverInfo = getFileHoverInformation("schema", schemaNode.getClassLeafIdentifierString(), schemaNode.getRange());
         if (hoverInfo.isEmpty()) {
             return null;
         }
         return hoverInfo.get();
     }
 
-    public static Optional<Hover> getFileHoverInformation(String markdownKey, Range range) {
+    public static Optional<Hover> getFileHoverInformation(String hoverDirectory, String markdownKey, Range range) {
         // avoid doing unnecessary IO operations
         if (markdownContentCache.containsKey(markdownKey)) {
             Optional<MarkupContent> mdContent = markdownContentCache.get(markdownKey);
@@ -350,8 +351,9 @@ public static Optional<Hover> getFileHoverInformation(String markdownKey, Range
 
         if (SchemaLanguageServer.serverPath == null)return Optional.empty();
         String fileName = markdownKey + ".md";
+        fileName = FileUtils.sanitizeFileName(fileName);
 
-        Path markdownPath = SchemaLanguageServer.serverPath.resolve("hover").resolve(fileName);
+        Path markdownPath = SchemaLanguageServer.serverPath.resolve("hover").resolve(hoverDirectory).resolve(fileName);
 
         try {
             String markdown = Files.readString(markdownPath);

integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/schemadocument/SchemaDocument.java

@@ -117,10 +117,7 @@ public void updateFileContent(String content) {
             //CSTUtils.printTree(logger, CST);
         }
 
-
-
         //schemaIndex.dumpIndex();
-
     }
 
     private List<Diagnostic> verifyFileName() {

integration/schema-language-server/language-server/src/main/java/ai/vespa/schemals/schemadocument/SchemaDocumentScheduler.java

@@ -75,6 +75,7 @@ private Optional<DocumentType> getDocumentTypeFromURI(String fileURI) {
     }
 
     public void updateFile(String fileURI, String content, Integer version) {
+        logger.info("Updating file: " + fileURI);
         Optional<DocumentType> documentType = getDocumentTypeFromURI(fileURI);
         if (documentType.isEmpty()) return;
 
@@ -195,27 +196,28 @@ public String getWorkspaceURI() {
     }
 
     public void openDocument(TextDocumentItem document) {
-        logger.info("Opening document: " + document.getUri());
+        String fileURI = FileUtils.decodeURL(document.getUri());
+        logger.info("Opening document: " + fileURI);
 
-        Optional<DocumentType> documentType = getDocumentTypeFromURI(document.getUri());
+        Optional<DocumentType> documentType = getDocumentTypeFromURI(fileURI);
 
         if (workspaceURI == null && documentType.isPresent() && (
             documentType.get() == DocumentType.SCHEMA ||
             documentType.get() == DocumentType.PROFILE
         )) {
-            Optional<URI> workspaceURI = FileUtils.findSchemaDirectory(URI.create(document.getUri()));
+            Optional<URI> workspaceURI = FileUtils.findSchemaDirectory(URI.create(fileURI));
             if (workspaceURI.isEmpty()) {
                 messageHandler.sendMessage(MessageType.Warning, 
-                    "The file " + document.getUri() + 
+                    "The file " + fileURI + 
                     " does not appear to be inside a 'schemas' directory. " + 
                     "Language support will be limited.");
             } else {
                 setupWorkspace(workspaceURI.get());
             }
         }
 
-        updateFile(document.getUri(), document.getText(), document.getVersion());
-        workspaceFiles.get(document.getUri()).setIsOpen(true);
+        updateFile(fileURI, document.getText(), document.getVersion());
+        workspaceFiles.get(fileURI).setIsOpen(true);
     }
 
     /*

integration/schema-language-server/lemminx-vespa/src/main/java/ai/vespa/lemminx/VespaExtension.java

@@ -1,5 +1,6 @@
 package ai.vespa.lemminx;
 
+import java.io.File;
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.util.logging.Logger;
@@ -44,7 +45,7 @@ public void doSave(ISaveContext context) { }
 	@Override
 	public void start(InitializeParams params, XMLExtensionsRegistry registry) {
         try {
-            serverPath = Paths.get(VespaExtension.class.getProtectionDomain().getCodeSource().getLocation().getPath()).getParent();
+            serverPath = Paths.get(new File(VespaExtension.class.getProtectionDomain().getCodeSource().getLocation().toURI()).getCanonicalPath()).getParent();
 
             UnpackRNGFiles.unpackRNGFiles(
                 serverPath