Merge branch 'GrammaticalFramework:master' into nheengatu-resource-grammar

Author: inariksit

.github/workflows/test.yml

@@ -0,0 +1,33 @@
+name: Check that the RGL can successfully build
+
+on:
+  - push
+  - pull_request
+
+jobs:
+  build:
+    runs-on: ubuntu-20.04
+    env:
+      GF_VERSION: 3.11
+      DEST: gf-rgl
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v2
+
+    - name: Download GF
+      uses: dsaltares/fetch-gh-release-asset@1.1.1
+      with:
+        repo: 'GrammaticalFramework/gf-core'
+        version: 'tags/${{ env.GF_VERSION }}'
+        file: 'gf-${{ env.GF_VERSION }}-ubuntu-20.04.deb'
+        token: ${{ secrets.GITHUB_TOKEN }}
+
+    - name: Install GF
+      run: |
+        sudo dpkg -i gf-${GF_VERSION}-ubuntu-20.04.deb
+
+    - name: Build RGL
+      run: |
+        mkdir -p ${DEST}
+        bash Setup.sh --dest=${DEST} --gf=gf --verbose

Setup.bat

@@ -37,20 +37,6 @@ if "%dest%"=="" (
 )
 :BreakLibPath
 
-set DATA_DIR=..\gf-core\DATA_DIR
-if "%dest%"=="" (
-  REM Look in already compiled GF folder
-  if exist %DATA_DIR% (
-    for /f "delims=" %%x in (%DATA_DIR%) do (
-      if not "%%x"=="" (
-        set dest=%%x\lib
-        goto BreakDataDir
-      )
-    )
-  )
-)
-:BreakDataDir
-
 if "%dest%"=="" (
   echo Unable to determine where to install the RGL. Please do one of the following:
   echo  - Pass the --dest=... flag to this script

Setup.hs

@@ -8,8 +8,8 @@ import System.IO (hPutStrLn,stderr)
 import System.IO.Error (catchIOError)
 import System.Exit (ExitCode(..),exitFailure)
 import System.Environment (getArgs,lookupEnv)
-import System.Process (rawSystem)
-import System.FilePath ((</>),splitSearchPath) -- ,takeFileName,addExtension,dropExtension)
+import System.Process (rawSystem,readProcess)
+import System.FilePath ((</>),splitSearchPath)
 import System.Directory (createDirectoryIfMissing,copyFile,getDirectoryContents,removeDirectoryRecursive,findFile)
 #if __GLASGOW_HASKELL__>=800
 import System.Directory (getModificationTime,setModificationTime)
@@ -61,7 +61,7 @@ errLocation = unlines $
   [ "Unable to determine where to install the RGL. Please do one of the following:"
   , " - Pass the " ++ destination_flag ++ "... flag to this script"
   , " - Set the GF_LIB_PATH environment variable"
-  , " - Compile & install GF from the gf-core repository (must be in same directory as gf-rgl)"
+  , " - Compile & install GF from the gf-core repository"
   ]
 
 -- | Copy single file between directories
@@ -116,10 +116,11 @@ data Info = Info
 mkInfo :: IO Info
 mkInfo = do
   args <- getArgs
+  let gf = maybe default_gf id (getFlag gf_flag args)
   -- Look for install location in a few different places
   let mflag = getFlag destination_flag args
-  mbuilt <- catchIOError (readFile "../gf-core/DATA_DIR" >>= \d -> return (Just (d </> "lib"))) (\e -> return Nothing)
   menvar <- lookupEnv "GF_LIB_PATH" >>= return . fmap (head . splitSearchPath)
+  mbuilt <- catchIOError (readProcess gf ["--version"] "" >>= return . getPath) (\e -> return Nothing)
   let
     inst_dir =
       case catMaybes [mflag,menvar,mbuilt] of
@@ -129,12 +130,18 @@ mkInfo = do
   return $ Info
     { infoBuildDir = "dist"
     , infoInstallDir = inst_dir
-    , infoGFPath = maybe default_gf id (getFlag gf_flag args)
+    , infoGFPath = gf
     , infoVerbose = verbose
     }
   where
     default_gf = "gf"
 
+    getPath s =
+      let l = last (lines s)
+      in if take 14 l == "Shared folder:"
+           then Just (drop 15 l </> "lib")
+           else Nothing
+
 getRGLBuildDir :: Info -> Mode -> FilePath
 getRGLBuildDir info mode = infoBuildDir info </> getRGLBuildSubDir mode
 

Setup.sh

@@ -38,15 +38,15 @@ done
 if [ -z "$dest" ]; then
   dest=$(echo "$GF_LIB_PATH" | sed 's/:.*$//')
 fi
-if [ -z "$dest" ] && [ -f "../gf-core/DATA_DIR" ]; then
-  dest=$(cat ../gf-core/DATA_DIR)
+if [ -z "$dest" ] && [ "$(gf --version | tail -1 | cut -c -14)" == "Shared folder:" ]; then
+  dest=$(gf --version | tail -1 | cut -c 16-)
   if [ -n "$dest" ]; then dest="${dest}/lib"; fi
 fi
 if [ -z "$dest" ]; then
   echo "Unable to determine where to install the RGL. Please do one of the following:"
   echo " - Pass the --dest=... flag to this script"
   echo " - Set the GF_LIB_PATH environment variable"
-  echo " - Compile & install GF from the gf-core repository (must be in same directory as gf-rgl)"
+  echo " - Compile & install GF from the gf-core repository"
   exit 1
 fi
 

doc/missing-in-rgl-Grammar.tsv

@@ -159,7 +159,7 @@ The GF Resource Grammar Library.
 //Linguistic Issues in Language Technology//,
 2 (2),
 2009.
-[PDF http://elanguage.net/journals/index.php/lilt/article/viewFile/214/158]
+[PDF https://journals.colorado.edu/index.php/lilt/article/view/1205/1047]
 #BR
 //A systematic presentation of the library from the linguistic point of view.//
 //Not only about English, but English examples abound.//

doc/rgl-publications.txt

@@ -0,0 +1,93 @@
+# show missing functions in the RGL languages
+# basic usage: python3 show_missing.py >missing-in-rgl-Grammar.tsv
+
+import subprocess
+
+# this is the functions you want to find: default is all funs in Grammar
+GF_LIB_PATH = '/Users/aarne/GF/dist/build/rgl/'
+ALLTENSES_PATH = GF_LIB_PATH + 'alltenses/'
+RGL_SOURCE_PATH = '/Users/aarne/GF/gf-rgl/src/'
+GRAMMAR = 'Grammar'
+
+# these are the languages you investigate
+LANGS = ['Cze', 'Urd']
+
+# get all functions in GRAMMAR, together with their types
+def get_funs(module=GRAMMAR):
+    cmd = 'echo "pg -funs" | gf -run ' + ALLTENSES_PATH + module + '.gfo'
+    missing = subprocess.Popen(cmd, text=True, shell=True,
+                               stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+    output = missing.communicate()[0].split(';')
+    return ([(fun.split()[0].strip(), ' '.join(fun.split()[2:]))  for fun in output if fun.split()])
+
+# get all missing functions in GRAMMARLng; this can take a long time
+def get_missing_from_compiled(Lng, module=GRAMMAR):
+    print('investigating', Lng)
+    cmd = 'echo "pg -missing" | gf -run ' + ALLTENSES_PATH + module + Lng + '.gfo'
+    missing = subprocess.Popen(cmd, text=True, shell=True,
+                               stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+    output = missing.communicate()[0].split()
+    return set(output[2:])
+
+# to get all languages in the compiled RGL, use all_langs from here
+def all_rgl_compiled_langs():
+    modules = subprocess.run(['ls', ALLTENSES_PATH], capture_output=True, text=True)
+    files = str(modules.stdout)
+    return [file[-7:-4] for file in files.split() if file[:-7].endswith(GRAMMAR)]
+
+# LANGS = all_rgl_compiled_langs()  # uncomment this line if you want all languages
+
+
+# it is much faster to use source files (sending gfos to /tmp)
+def str_until(c, s):
+    i = s.find(c)
+    if i >= 0:
+        return s[:i]
+    else:
+        return s
+    
+def get_missing_from_source(lang, gf_file):
+    cmd = 'gf -batch -retain -no-pmcfg -src -gfo-dir=/tmp ' + gf_file
+    missing = subprocess.Popen(cmd, text=True, shell=True,
+                               stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    output = missing.communicate()[1].split('\n')
+    missing = (lang, {str_until('\x1b[39;49m', line.split()[-1])
+                      for line in output
+                      if line.strip().startswith('Warning: no linearization of')})
+    return missing
+
+
+def all_rgl_source_modules(module=GRAMMAR):
+    cmd = 'ls ' + RGL_SOURCE_PATH + '*/' + module + '?*.gf'
+    files = subprocess.Popen(cmd, text=True, shell=True,
+                               stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+    output = files.communicate()[0].split()
+    return {file[-6:-3]: file for file in output}
+
+LANGS = all_rgl_source_modules()  # Lng: file dict
+
+# make a text with tab-separated strings withcolumns fun, type, langs and rows for each fun
+def make_table(funs=get_funs(), module=GRAMMAR, langs=LANGS):
+    header = ['fun', 'type'] + list(langs.keys())  # if dict, otherwise langs ###
+    rows = [header]
+    print('\t'.join(header))
+    
+#    missings = {lang: get_missing_from_compiled(lang, module) for lang in langs}
+    missings = {lang: get_missing_from_source(lang, file)[1] for lang,file in langs.items()}
+    for fun in funs:
+        row = [fun[0], fun[1]]
+        for lang in langs.keys():
+            if fun[0] in missings[lang]:
+                row.append('-')
+            else:
+                row.append('+')
+        rows.append(row)
+        print('\t'.join(row))
+
+
+# the output is send to stdout
+make_table()
+
+
+    
+

doc/show_missing.py

@@ -7,8 +7,6 @@ Aarne Ranta
 The following table gives the languages currently available in the
 GF Resource Grammar Library.
 
-For another view, see the
-[The Resource Grammar Library coverage map http://www.postcrashgames.com/gf_world/] .
 
 Corrections and additions are welcome! Notice that only those parts of implementations
 that are currently available via https://github.com/GrammaticalFramework/gf-rgl/

doc/status.txt

@@ -32,7 +32,6 @@ the library, view all functions in a module's scope, and quickly jump to their d
 Other relevant documents:
 - [The RGL Status Document ../status.html]: the current status of different languages
   and the authors of each grammar
-- [The Resource Grammar Library coverage map http://www.postcrashgames.com/gf_world/]
 - [RGL Documentation and Publications ../rgl-publications.html]: links to publications and other documentation
 - [More modules ../gfdoc/sources.html]: extra modules, dictionaries, and
   the internals of the resource grammar
@@ -46,7 +45,7 @@ abstract functions and their Universal Dependency labels
 - Paper "The GF Resource Grammar Library" by A. Ranta
   (//Linguistic Issues in Language Technology//, 2 (2), 2009). An overview of
   the library with linguistic motivations.
-  [PDF http://journals.linguisticsociety.org/elanguage/lilt/article/download/214/214-501-1-PB.pdf]
+  [PDF https://journals.colorado.edu/index.php/lilt/article/view/1205/1047]
 - Paper "Grammars as Software Libraries" by A. Ranta
   (In Y. Bertot, G. Huet, J-J. Lévy, and G. Plotkin (eds.),
    //From Semantics to Computer Science//, Cambridge University Press,