WIP

FHIR · Oct 24, 2024 · 94c59fc · 94c59fc
1 parent 94c1d04
commit 94c59fc
Show file tree

Hide file tree

Showing 22 changed files with 365 additions and 371 deletions.
diff --git a/README.md b/README.md
@@ -116,8 +116,6 @@ if (status !== LoadStatus.LOADED) {
 }
 ```
 
-To instantiate the default `PackageLoader` with a set of standalone JSON or XML resources that should be pre-loaded, use the `defaultPackageLoaderWithLocalResources` function instead, passing in an array of file paths to folders containing the resources to load.
-
 For more control over the `PackageLoader`, use the [BasePackageLoader](src/loader/BasePackageLoader.ts). This allows you to specify the [PackageDB](src/db), [PackageCache](src/cache), [RegistryClient](src/registry), and [CurrentBuildClient](src/current) you wish to use. FHIRPackageLoader comes with implementations of each of these, but you may also provide your own implementations that adhere to the relevant interfaces.
 
 ### PackageLoader Functions

diff --git a/src/app.ts b/src/app.ts
@@ -37,7 +37,7 @@ async function install(fhirPackages: string[], options: OptionValues) {
   const SQL = await initSqlJs();
   const packageDB = new SQLJSPackageDB(new SQL.Database());
   const fhirCache = options.cachePath ?? path.join(os.homedir(), '.fhir', 'packages');
-  const packageCache = new DiskBasedPackageCache(fhirCache, [], { log });
+  const packageCache = new DiskBasedPackageCache(fhirCache, { log });
   const registryClient = new DefaultRegistryClient({ log });
   const buildClient = new BuildDotFhirDotOrgClient({ log });
   const loader = new BasePackageLoader(packageDB, packageCache, registryClient, buildClient, {

diff --git a/src/cache/DiskBasedPackageCache.ts b/src/cache/DiskBasedPackageCache.ts
@@ -1,12 +1,7 @@
 import path from 'path';
 import fs from 'fs-extra';
 import { LogFunction } from '../utils';
-import {
-  PackageCache,
-  PackageCacheOptions,
-  LOCAL_PACKAGE_NAME,
-  LOCAL_PACKAGE_VERSION
-} from './PackageCache';
+import { PackageCache, PackageCacheOptions } from './PackageCache';
 import temp from 'temp';
 import * as tar from 'tar';
 import { Readable } from 'stream';
@@ -16,17 +11,14 @@ import { Fhir as FHIRConverter } from 'fhir/fhir';
 
 export class DiskBasedPackageCache implements PackageCache {
   private log: LogFunction;
-  private localResourceFolders: string[];
   private fhirConverter: FHIRConverter;
   private lruCache: LRUCache<string, any>;
 
   constructor(
     private cachePath: string,
-    localResourceFolders: string[] = [],
     options: PackageCacheOptions = {}
   ) {
     this.log = options.log ?? (() => {});
-    this.localResourceFolders = localResourceFolders.map(f => path.resolve(f));
     this.fhirConverter = new FHIRConverter();
     // TODO: Make Cache Size Configurable
     this.lruCache = new LRUCache<string, any>(500);
@@ -51,32 +43,19 @@ export class DiskBasedPackageCache implements PackageCache {
   }
 
   isPackageInCache(name: string, version: string): boolean {
-    if (isLocalPackage(name, version)) {
-      return true;
-    }
     return fs.existsSync(path.resolve(this.cachePath, `${name}#${version}`));
   }
 
   getPackagePath(name: string, version: string): string | undefined {
     if (this.isPackageInCache(name, version)) {
-      if (isLocalPackage(name, version)) {
-        return this.localResourceFolders.join(';');
-      }
       return path.resolve(this.cachePath, `${name}#${version}`);
     }
   }
 
   getPackageJSONPath(name: string, version: string): string | undefined {
-    if (!isLocalPackage(name, version)) {
-      const jsonPath = path.resolve(
-        this.cachePath,
-        `${name}#${version}`,
-        'package',
-        'package.json'
-      );
-      if (fs.existsSync(jsonPath)) {
-        return jsonPath;
-      }
+    const jsonPath = path.resolve(this.cachePath, `${name}#${version}`, 'package', 'package.json');
+    if (fs.existsSync(jsonPath)) {
+      return jsonPath;
     }
   }
 
@@ -85,77 +64,11 @@ export class DiskBasedPackageCache implements PackageCache {
       return [];
     }
 
-    if (isLocalPackage(name, version)) {
-      const spreadSheetCounts = new Map<string, number>();
-      const invalidFileCounts = new Map<string, number>();
-      const resourcePaths: string[] = [];
-      this.localResourceFolders.forEach(folder => {
-        try {
-          let spreadSheetCount = 0;
-          let invalidFileCount = 0;
-          fs.readdirSync(folder, { withFileTypes: true })
-            .filter(entry => {
-              if (!entry.isFile()) {
-                return false;
-              } else if (/\.json$/i.test(entry.name)) {
-                return true;
-              } else if (/-spreadsheet.xml/i.test(entry.name)) {
-                spreadSheetCount++;
-                this.log(
-                  'debug',
-                  `Skipped spreadsheet XML file: ${path.resolve(entry.path, entry.name)}`
-                );
-                return false;
-              } else if (/\.xml/i.test(entry.name)) {
-                const xml = fs.readFileSync(path.resolve(entry.path, entry.name)).toString();
-                if (/<\?mso-application progid="Excel\.Sheet"\?>/m.test(xml)) {
-                  spreadSheetCount++;
-                  this.log(
-                    'debug',
-                    `Skipped spreadsheet XML file: ${path.resolve(entry.path, entry.name)}`
-                  );
-                  return false;
-                }
-                return true;
-              }
-              invalidFileCount++;
-              this.log(
-                'debug',
-                `Skipped non-JSON / non-XML file: ${path.resolve(entry.path, entry.name)}`
-              );
-              return false;
-            })
-            .forEach(entry => resourcePaths.push(path.resolve(entry.path, entry.name)));
-          spreadSheetCounts.set(folder, spreadSheetCount);
-          invalidFileCounts.set(folder, invalidFileCount);
-        } catch {
-          this.log('error', `Failed to load resources from local path: ${folder}`);
-        }
-      });
-      spreadSheetCounts.forEach((count, folder) => {
-        if (count) {
-          this.log(
-            'info',
-            `Found ${count} spreadsheet(s) in directory: ${folder}. SUSHI does not support spreadsheets, so any resources in the spreadsheets will be ignored. To see the skipped files in the logs, run SUSHI with the "--log-level debug" flag.`
-          );
-        }
-      });
-      invalidFileCounts.forEach((count, folder) => {
-        if (count) {
-          this.log(
-            'info',
-            `Found ${count} non-JSON / non-XML file(s) in directory: ${folder}. SUSHI only processes resource files with JSON or XML extensions. To see the skipped files in the logs, run SUSHI with the "--log-level debug" flag.`
-          );
-        }
-      });
-      return resourcePaths;
-    } else {
-      const contentPath = path.resolve(this.cachePath, `${name}#${version}`, 'package');
-      return fs
-        .readdirSync(contentPath, { withFileTypes: true })
-        .filter(entry => entry.isFile() && /^[^.].*\.json$/i.test(entry.name))
-        .map(entry => path.resolve(entry.path, entry.name));
-    }
+    const contentPath = path.resolve(this.cachePath, `${name}#${version}`, 'package');
+    return fs
+      .readdirSync(contentPath, { withFileTypes: true })
+      .filter(entry => entry.isFile() && /^[^.].*\.json$/i.test(entry.name))
+      .map(entry => path.resolve(entry.path, entry.name));
   }
 
   getResourceAtPath(resourcePath: string) {
@@ -183,10 +96,6 @@ export class DiskBasedPackageCache implements PackageCache {
   }
 }
 
-function isLocalPackage(name: string, version: string) {
-  return name === LOCAL_PACKAGE_NAME && version === LOCAL_PACKAGE_VERSION;
-}
-
 /**
  * This function takes a package which contains contents at the same level as the "package" folder, and nests
  * all that content within the "package" folder.

diff --git a/src/cache/PackageCache.ts b/src/cache/PackageCache.ts
@@ -1,9 +1,6 @@
 import { Readable } from 'stream';
 import { LogFunction } from '../utils';
 
-export const LOCAL_PACKAGE_NAME = 'LOCAL';
-export const LOCAL_PACKAGE_VERSION = 'LOCAL';
-
 export type PackageCacheOptions = {
   log?: LogFunction;
 };

diff --git a/src/db/PackageDB.ts b/src/db/PackageDB.ts
@@ -9,4 +9,5 @@ export interface PackageDB {
   findResourceInfos(key: string, options?: FindResourceInfoOptions): ResourceInfo[];
   findResourceInfo(key: string, options?: FindResourceInfoOptions): ResourceInfo | undefined;
   getPackageStats(name: string, version: string): PackageStats | undefined;
+  exportDB(): Promise<{ mimeType: string; data: Buffer }>;
 }
diff --git a/src/db/SQLJSPackageDB.ts b/src/db/SQLJSPackageDB.ts
@@ -318,6 +318,11 @@ export class SQLJSPackageDB implements PackageDB {
     };
   }
 
+  exportDB(): Promise<{ mimeType: string; data: Buffer }> {
+    const data = this.db.export();
+    return Promise.resolve({ mimeType: 'application/x-sqlite3', data: Buffer.from(data) });
+  }
+
   logPackageTable() {
     const res = this.db.exec('SELECT * FROM package');
     console.log(util.inspect(res, false, 3, true));

diff --git a/src/loader/BasePackageLoader.ts b/src/loader/BasePackageLoader.ts
@@ -31,6 +31,13 @@ export class BasePackageLoader implements PackageLoader {
   async loadPackage(name: string, version: string): Promise<LoadStatus> {
     let packageLabel = `${name}#${version}`;
 
+    const originalVersion = version;
+    version = await this.registryClient.resolveVersion(name, version);
+    if (version !== originalVersion) {
+      this.log('info', `Resolved ${packageLabel} to concrete version ${version}`);
+      packageLabel = `${name}#${version}`;
+    }
+
     // If it's already loaded, then there's nothing to do
     if (this.getPackageLoadStatus(name, version) === LoadStatus.LOADED) {
       this.log('info', `${packageLabel} already loaded`);
@@ -47,6 +54,7 @@ export class BasePackageLoader implements PackageLoader {
       packageLabel = `${name}#${version}`;
     }
 
+    let downloadErrorMessage: string;
     // If it's a "current" version, download the latest version from the build server (if applicable)
     if (isCurrentVersion(version)) {
       const branch = version.indexOf('$') !== -1 ? version.split('$')[1] : undefined;
@@ -55,7 +63,7 @@ export class BasePackageLoader implements PackageLoader {
           const tarballStream = await this.currentBuildClient.downloadCurrentBuild(name, branch);
           await this.packageCache.cachePackageTarball(name, version, tarballStream);
         } catch {
-          this.log('error', `Failed to download ${packageLabel} from current builds`);
+          downloadErrorMessage = `Failed to download most recent ${packageLabel} from current builds`;
         }
       }
     }
@@ -65,7 +73,7 @@ export class BasePackageLoader implements PackageLoader {
         const tarballStream = await this.registryClient.download(name, version);
         await this.packageCache.cachePackageTarball(name, version, tarballStream);
       } catch {
-        this.log('error', `Failed to download ${packageLabel} from registry`);
+        downloadErrorMessage = `Failed to download ${packageLabel} from the registry`;
       }
     }
 
@@ -74,9 +82,17 @@ export class BasePackageLoader implements PackageLoader {
     try {
       stats = this.loadPackageFromCache(name, version);
     } catch {
-      this.log('error', `Failed to load ${name}#${version}`);
+      this.log(
+        'error',
+        `Failed to load ${packageLabel}${downloadErrorMessage ? `: ${downloadErrorMessage}` : ''}`
+      );
       return LoadStatus.FAILED;
     }
+    if (downloadErrorMessage) {
+      // Loading succeeded despite a download error. This might happen if a current build is stale,
+      // but the download fails, in which case the stale build will be loaded instead.
+      this.log('error', downloadErrorMessage);
+    }
     this.log('info', `Loaded ${stats.name}#${stats.version} with ${stats.resourceCount} resources`);
     return LoadStatus.LOADED;
   }
@@ -117,12 +133,13 @@ export class BasePackageLoader implements PackageLoader {
     this.packageDB.savePackageInfo(info);
 
     // Register the resources
-    await pkg.registerResources((key: string, resource: any) => {
+    await pkg.registerResources((key: string, resource: any, allowNonResources?: boolean) => {
       this.loadResource(
         `virtual:${packageKey}:${key}`,
         resource,
         packageJSON.name,
-        packageJSON.version
+        packageJSON.version,
+        allowNonResources
       );
     });
 
@@ -147,14 +164,9 @@ export class BasePackageLoader implements PackageLoader {
       version
     };
 
-    if (name === 'LOCAL' && version === 'LOCAL') {
-      // it's a list of ;-separated paths, so don't try to resolve it
-      info.packagePath = packagePath;
-    } else {
-      const packageJSONPath = this.packageCache.getPackageJSONPath(name, version);
-      info.packagePath = path.resolve(packagePath);
-      info.packageJSONPath = path.resolve(packageJSONPath);
-    }
+    const packageJSONPath = this.packageCache.getPackageJSONPath(name, version);
+    info.packagePath = path.resolve(packagePath);
+    info.packageJSONPath = path.resolve(packageJSONPath);
 
     this.packageDB.savePackageInfo(info);
 
@@ -172,7 +184,10 @@ export class BasePackageLoader implements PackageLoader {
           this.loadResourceFromCache(resourcePath, packageName, packageVersion);
         } catch {
           // swallow this error because some JSON files will not be resources
-          this.log('debug', `JSON file at path ${resourcePath} was not FHIR resource`);
+          // and don't log it if it is package.json (since every package should have one)
+          if (path.basename(resourcePath) !== 'package.json') {
+            this.log('debug', `JSON file at path ${resourcePath} was not FHIR resource`);
+          }
         }
       });
   }
@@ -186,14 +201,21 @@ export class BasePackageLoader implements PackageLoader {
     resourcePath: string,
     resourceJSON: any,
     packageName?: string,
-    packageVersion?: string
+    packageVersion?: string,
+    allowNonResources = false
   ) {
     // We require at least a resourceType in order to know it is FHIR
-    if (typeof resourceJSON.resourceType !== 'string' || resourceJSON.resourceType === '') {
-      throw new InvalidResourceError(resourcePath, 'resource does not specify its resourceType');
+    let resourceType = resourceJSON.resourceType;
+    if (typeof resourceType !== 'string' || resourceType === '') {
+      if (allowNonResources) {
+        // SUSHI needs to support registering instances of logical models, but some code expects resourceType
+        resourceType = 'Unknown';
+      } else {
+        throw new InvalidResourceError(resourcePath, 'resource does not specify its resourceType');
+      }
     }
 
-    const info: ResourceInfo = { resourceType: resourceJSON.resourceType };
+    const info: ResourceInfo = { resourceType };
     if (typeof resourceJSON.id === 'string') {
       info.id = resourceJSON.id;
     }
@@ -206,7 +228,7 @@ export class BasePackageLoader implements PackageLoader {
     if (typeof resourceJSON.version === 'string') {
       info.version = resourceJSON.version;
     }
-    if (resourceJSON.resourceType === 'StructureDefinition') {
+    if (resourceType === 'StructureDefinition') {
       if (typeof resourceJSON.kind === 'string') {
         info.sdKind = resourceJSON.kind;
       }
@@ -372,6 +394,10 @@ export class BasePackageLoader implements PackageLoader {
     }
   }
 
+  exportDB(): Promise<{ mimeType: string; data: Buffer }> {
+    return this.packageDB.exportDB();
+  }
+
   clear() {
     this.packageDB.clear();
   }

diff --git a/src/loader/DefaultPackageLoader.ts b/src/loader/DefaultPackageLoader.ts
@@ -10,17 +10,10 @@ import { BasePackageLoader, BasePackageLoaderOptions } from './BasePackageLoader
 // TODO: New options w/ option for overriding FHIR cache
 
 export async function defaultPackageLoader(options: BasePackageLoaderOptions) {
-  return defaultPackageLoaderWithLocalResources([], options);
-}
-
-export async function defaultPackageLoaderWithLocalResources(
-  localResourceFolders: string[],
-  options: BasePackageLoaderOptions
-) {
   const SQL = await initSqlJs();
   const packageDB = new SQLJSPackageDB(new SQL.Database());
   const fhirCache = path.join(os.homedir(), '.fhir', 'packages');
-  const packageCache = new DiskBasedPackageCache(fhirCache, localResourceFolders, {
+  const packageCache = new DiskBasedPackageCache(fhirCache, {
     log: options.log
   });
   const registryClient = new DefaultRegistryClient({ log: options.log });

diff --git a/src/loader/PackageLoader.ts b/src/loader/PackageLoader.ts
@@ -18,5 +18,6 @@ export interface PackageLoader {
   findResourceInfo(key: string, options?: FindResourceInfoOptions): ResourceInfo | undefined;
   findResourceJSONs(key: string, options?: FindResourceInfoOptions): any[];
   findResourceJSON(key: string, options?: FindResourceInfoOptions): any | undefined;
+  exportDB(): Promise<{ mimeType: string; data: Buffer }>;
   clear(): void;
 }