Merge pull request #2 from cgay/dev

Various and sundry getting started mods

.gitmodules

@@ -1,7 +1,18 @@
-[submodule "json"]
-	path = json
-	url = git@github.com:pedro-w/json.git
-	branch = updates-for-lsp
 [submodule "vscode-dylan"]
 	path = vscode-dylan
 	url = git@github.com:pedro-w/vscode-dylan.git
+[submodule "ext/json"]
+	path = ext/json
+	url = git@github.com:dylan-lang/json
+[submodule "ext/workspaces"]
+	path = ext/workspaces
+	url = git@github.com:cgay/workspaces
+[submodule "ext/uncommon-dylan"]
+	path = ext/uncommon-dylan
+	url = git@github.com:cgay/uncommon-dylan
+[submodule "ext/pacman"]
+	path = ext/pacman
+	url = git@github.com:cgay/pacman
+[submodule "ext/command-line-parser"]
+	path = ext/command-line-parser
+	url = git@github.com:dylan-lang/command-line-parser

README.md

@@ -2,30 +2,79 @@
 
 This is an implementation of the [Language Server
 Protocol](https://microsoft.github.io/language-server-protocol/) for
-Dylan.  At the moment it will respond to the initialize/ shutdown
-sequence but does not actually implement any of the methods defined in
-the LSP.  Note that this project includes a git submodule for
-json. This adds `null` handling and a method to output json as a
-string to the open-dylan/json project.
+Dylan.
 
-We are currently using version [3.15 of the LSP protocol](https://microsoft.github.io/language-server-protocol/specifications/specification-3-15/)
 
-## Usage
+## Current Status
+
+As of 2021.03.14, the only function fully implemented is "jump to definition"
+and (at least in Emacs) when you jump to another `.dylan` file, that file does
+not in automatically have LSP enabled so you must use `M-x lsp` again.
+
+We are currently using version [3.15 of the LSP protocol](https://microsoft.github.io/language-server-protocol/specifications/specification-3-15/).
+
+
+## Opening Projects
+
+The LSP server needs to be able to open a project (essentially a Dylan library)
+associated with the file you're editing when you turn on LSP in your editor. It
+makes two attempts, in the following order:
+
+1. Search up in the directory structure until it finds a `workspace.json` file,
+   which indicates a [dylan-tool](https://github.com/cgay/dylan-tool)
+   workspace.  In this case it looks for the "default-library" setting in the
+   workspace file and opens that library.  If there is no default library set
+   and there is only one "active" library, it uses that. Otherwise it fails.
+
+2. Search up in the directory structure for a `registry` directory and open the
+   library associated with the first `*.lid` file it finds there. (Note that it
+   currently removes the `.lid` suffix and assumes that a library by the same
+   name as the basename of the file can be opened via the registry. This should
+   eventually be fixed.)
+
+
+## Emacs Usage
 
 Testing with Emacs [lsp-mode](https://github.com/emacs-lsp/lsp-mode).
 
 1. Install lsp-mode (see github project page for details)
-2. Start emacs with `emacs --load=setup.el` in this directory
-3. Open a Dylan file
-4. Type `M-x lsp` to start the client, which will connect to the server
 
-The file `setup.el` is used just to avoid making any changes to the
-user's `.emacs`.
+2. Set environment variables.
+
+   a. Currently the LSP server only opens projects via the Dylan registry so
+      it's important to either start emacs in the directory containing your
+      "registry" directory or set `OPEN_DYLAN_USER_REGISTRIES` to contain that
+      registry directory.
 
-You must set `OPEN_DYLAN_RELEASE` and `OPEN_DYLAN_USER_REGISTRIES` appropriately.
-Currently the only function is `lsp-find-definition` which will jump to the definition of the symbol under the cursor. Unfortunately it is still not reliable and depends on some hard-coded defaults.
+      If you are developing the lsp-dylan code itself and also modifying Open
+      Dylan at the same time, you may want to include
+      `.../opendylan/sources/registry` in the list as well. Otherwise, when you
+      use `M-.` etc to jump to definitions for used libraries, files in the
+      Open Dylan install directory (which is not under source control) will be
+      opened. For example:
 
-Testing with VS Code (1.45.0 on macos)
+        export OPEN_DYLAN_USER_REGISTRIES=/home/you/lsp-dylan/registry:/home/you/opendylan/sources/registry
+
+   b. Point `OPEN_DYLAN_RELEASE_INSTALL` at the Open Dylan installation
+      directory. This is necessary so that it can find the Jam build scripts,
+      and core libraries. For example:
+
+        export OPEN_DYLAN_RELEASE_INSTALL=/home/you/opendylan-2021.1
+
+3. Start emacs and make sure that `setup.el` is loaded. For example:
+
+     `emacs --load=/home/you/lsp-dylan/setup.el`
+
+   Obviously you may modify your Emacs init file instead, if you prefer.
+
+4. Open a Dylan source file and type `M-x lsp` to start the client. The client
+   starts the LSP server (the `lsp-dylan` executable) and connects to it.
+
+   Currently `lsp-dylan` must be in `./_build/bin/lsp-dylan` or
+   `${DYLAN}/workspaces/lsp/_build/bin/lsp-dylan`. (TODO: search for it on
+   `PATH`.)
+
+## VS Code Usage (1.45.0 on macos)
 
 1. Open the `vscode` folder in VS Code
 1. First time only, `npm install` to get the dependencies
@@ -34,5 +83,16 @@ Testing with VS Code (1.45.0 on macos)
 4. A new VS Code window will open with the extension running.
 
 
+## References
+
+* [Intro to LSP from
+  Microsoft](https://docs.microsoft.com/en-us/visualstudio/extensibility/language-server-protocol)
+  Besides being a quick introduction, this has links to some other tools that
+  would help in developing VS Code integration for Dylan.
 
+* [LSP v3.15
+  Specification](https://microsoft.github.io/language-server-protocol/specifications/specification-3-15/)
+  This is the version we are currently coding to.
 
+* [langserver.org](https://langserver.org/) lists LSP implementations that
+  support at least one of the six major LSP features.

compiler.dylan

@@ -3,18 +3,19 @@ Synopsis: Communicaton with the Dylan command-line compiler
 Author: Peter
 Copyright: 2019
 
-/* The basis of this code is taken from the dswank module
-* author:    Andreas Bogk and Hannes Mehnert
-* copyright: Original Code is Copyright (c) 2008-2012 Dylan Hackers; All rights reversed.
-*/
+// The basis of this code is taken from the dswank module.
+// Author:    Andreas Bogk and Hannes Mehnert
+// Copyright: Original Code is Copyright (c) 2008-2012 Dylan Hackers; All rights reversed.
+
 
 define variable *server* = #f;
 define variable *project* = #f;
 define variable *module* = #f;
 define variable *library* = #f;
 define variable *project-name* = #f;
 
-define function start-compiler(input-stream, output-stream)
+define function start-compiler
+    (input-stream, output-stream) => (server :: <command-line-server>)
   make-environment-command-line-server(input-stream: input-stream,
                                        output-stream: output-stream)
 end function;
@@ -24,19 +25,19 @@ define function run-compiler(server, string :: <string>) => ()
   execute-command-line(server, string);
 end function run-compiler;
 
-/* Ask the command line compiler to open a project.
- * Param: server - the command line server
- * Param: name - the project name (either a registry name or a lid file)
- * Returns: the project object. (instance of <project-object>)
-*/
-define function open-project(server, name :: <string>)
- => (project :: <object>)
+// Ask the command line compiler to open a project.
+// Param: server - the command line server.
+// Param: name - either a library name or a lid file.
+// Returns: an instance of <project-object>
+define function open-project
+    (server, name :: <string>) => (project :: <object>)
   let command = make-command(<open-project-command>,
                              server: server.server-context,
                              file: as(<file-locator>, name));
   let project = execute-command(command);
-  local-log("Result of opening %s is %=\n", name, project);
-  local-log("Result of find %s is %=\n", project-name(project),
+  local-log("Result of opening %s is %=", name, project);
+  local-log("Result of find %s is %=",
+            project-name(project),
             find-project(project-name(project)));
   project
 end function;
@@ -47,12 +48,13 @@ define function describe-symbol (symbol-name)
   environment-object-description(*project*, env, *module*)
 end;
 
-define function symbol-location (symbol-name, #key module = #f)
-  let env = symbol-name & get-environment-object(symbol-name, module: module);
+define function symbol-location
+    (symbol-name :: <string>, #key module)
+  let env = get-environment-object(symbol-name, module: module);
   if (env)
     environment-object-source-location(*project*, env)
   else
-    local-log("No environment object for %s in module %s\n", symbol-name, module);
+    local-log("No environment object for %s in module %s", symbol-name, module);
   end
 end function;
 
@@ -61,7 +63,7 @@ define function list-all-package-names ()
             (dir :: <pathname>, filename :: <string>, type :: <file-type>)
           if (type == #"file")
             if (last(filename) ~= '~')
-              local-log("%s\n", filename);
+              local-log("%s", filename);
             end;
           end;
         end;
@@ -72,27 +74,27 @@ define function list-all-package-names ()
       do-directory(collect-project, reg-path);
     end;
   end;
-end;
+end function;
 
-define function one-off-debug()
+define function one-off-debug ()
   //list-all-package-names();
   let in-stream = make(<string-stream>);
   let out-stream = make(<string-stream>, direction: #"output");
   let srv = start-compiler(in-stream, out-stream);
   let project = open-project(srv, "testproject");
-  local-log("Database: %=\n", project-compiler-database(project));
+  local-log("Database: %=", project-compiler-database(project));
 
   *project* := project;
   *server* := srv;
   let symbol-name = "zeor";
   let library = project-library(project);
   let module = find-module(project, "testproject", library: library);
-let loc = environment-object-source-location(project, module).source-location-source-record;
+  let loc = environment-object-source-location(project, module).source-location-source-record;
   let env = find-environment-object(project,
                                     symbol-name,
                                     library: library,
                                     module: module);
-  local-log("one-off-debug:\nfind-environment-object(%s, %s, library:%s, module:%s) => %=\n",
+  local-log("one-off-debug:\n  find-environment-object(%s, %s, library:%s, module:%s) => %=",
             n(project),
             n(symbol-name),
             n(library),
@@ -103,47 +105,46 @@ let loc = environment-object-source-location(project, module).source-location-so
 //  fp := "testproject.dylan";
 //  fp := pfl;
   let (m, l) = file-module(project, fp);
-  local-log("one-off-debug:\n1. %=\n2. %=\n3. %=\n", n(fp), n(m), n(l));
+  local-log("one-off-debug:\n  1. %=\n  2. %=\n3. %=", n(fp), n(m), n(l));
   let same? = pfl = fp;
-  local-log("one-off-debug:\n1. %=\n2. %=\n",
+  local-log("one-off-debug:\n  1. %=\n  2. %=\n",
             locator-relative?(fp),
             locator-relative?(pfl));
-  local-log("one-off-debug:\n1. %=\n2. %=\n",
+  local-log("one-off-debug:\n  1. %=\n  2. %=\n",
             locator-base(fp),
             locator-base(pfl));
-  local-log("one-off-debug:\n1. %=\n2. %=\n",
+  local-log("one-off-debug:\n  1. %=\n  2. %=\n",
             locator-extension(fp),
             locator-extension(pfl));
-  local-log("one-off-debug:\n1. %=\n2. %=\n3.%=\n",
+  local-log("one-off-debug:\n  1. %=\n  2. %=\n  3.%=\n",
             locator-path(fp),
             locator-path(pfl),
             same?);
-end;
+end function;
 
 define method n (x :: <environment-object>)
   // for debugging!
   let s = print-environment-object-to-string(*project*, x);
   format-to-string("%s%s", object-class(x), s);
 end;
+
 define method n (x :: <string>)
   format-to-string("\"%s\"", x)
 end;
+
 define method n (x :: <locator>)
   format-to-string("locator:\"%s\"", as(<string>, x))
 end;
-define method n ( x == #f)
+
+define method n (x == #f)
   "#f"
 end;
-define function get-environment-object (symbol-name, #key module = #f)
-  let library = project-library(*project*);
-  unless (module)
-    // TODO not hard code
-    module := find-module(*project*, "testproject", library: library);
-  end;
-  local-log("%s -> module is %s\n", symbol-name, n(module));
 
+define function get-environment-object
+    (symbol-name :: <string>, #key module) => (o :: false-or(<environment-object>))
+  let library = project-library(*project*);
+  local-log("%s -> module is %s", symbol-name, n(module));
   find-environment-object(*project*, symbol-name,
-                                    library: library,
-                                    module: module);
-end;
-
+                          library: library,
+                          module: module);
+end function;

env.sh

@@ -1,5 +1,5 @@
 #!/bin/sh
 
 export OPEN_DYLAN_USER_REGISTRIES=/Users/peterhull/registry/
-export OPEN_DYLAN_RELEASE=/opt/local/2020.1pre/
+export OPEN_DYLAN_RELEASE_INSTALL=/opt/local/2020.1pre/