First pass at allowing translator to accept translated documents

packages/malloy/src/lang/ast/statements/import-statement.ts

@@ -97,8 +97,9 @@ export class ImportStatement
         'Cannot import without translation context'
       );
     } else if (this.fullURL) {
+      const pretranslated = trans.root.pretranslatedModels.get(this.fullURL);
       const src = trans.root.importZone.getEntry(this.fullURL);
-      if (src.status === 'present') {
+      if (pretranslated || src.status === 'present') {
         const importable = trans.getChildExports(this.fullURL);
         if (this.notEmpty()) {
           // just import the named objects

packages/malloy/src/lang/ast/types/document-compile-result.ts

@@ -21,13 +21,10 @@
  * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  */
 
-import {ModelDef, Query, SQLSourceDef} from '../../../model/malloy_types';
+import {DocumentDef} from '../../../model/malloy_types';
 
 import {ModelDataRequest} from '../../translate-response';
 
-export interface DocumentCompileResult {
-  modelDef: ModelDef;
-  queryList: Query[];
-  sqlBlocks: SQLSourceDef[];
+export interface DocumentCompileResult extends DocumentDef {
   needs: ModelDataRequest;
 }

packages/malloy/src/lang/parse-malloy.ts

@@ -41,6 +41,7 @@ import {
   Query,
   SourceDef,
   SQLSourceDef,
+  DocumentDef,
 } from '../model/malloy_types';
 import {MalloyLexer} from './lib/Malloy/MalloyLexer';
 import {MalloyParser} from './lib/Malloy/MalloyParser';
@@ -98,7 +99,8 @@ export type StepResponses =
   | ASTResponse
   | TranslateResponse
   | ParseResponse
-  | MetadataResponse;
+  | MetadataResponse
+  | PretranslatedResponse;
 
 /**
  * This ignores a -> popMode when the mode stack is empty, which is a hack,
@@ -132,8 +134,15 @@ interface TranslationStep {
 interface ParseData extends ProblemResponse, NeedURLData, FinalResponse {
   parse: MalloyParseInfo;
 }
+
 export type ParseResponse = Partial<ParseData>;
 
+interface PretranslatedData {
+  translation: DocumentDef;
+}
+
+export type PretranslatedResponse = PretranslatedData | null;
+
 /**
  * ParseStep -- Parse the source URL
  */
@@ -151,19 +160,6 @@ class ParseStep implements TranslationStep {
       return this.response;
     }
 
-    if (that.urlIsFullPath === undefined) {
-      try {
-        const _checkFull = new URL(that.sourceURL);
-        that.urlIsFullPath = true;
-      } catch (e) {
-        const msg = e instanceof Error ? e.message : '';
-        that.urlIsFullPath = false;
-        that.root.logError(
-          'failed-to-compute-absolute-import-url',
-          `Could not compute full path URL: ${msg}`
-        );
-      }
-    }
     if (!that.urlIsFullPath) {
       return that.fatalResponse();
     }
@@ -348,8 +344,10 @@ class ImportsAndTablesStep implements TranslationStep {
       allMissing = {tables};
     }
 
-    const missingImports = that.root.importZone.getUndefined();
-    if (missingImports) {
+    const missingImports = (that.root.importZone.getUndefined() ?? []).filter(
+      url => that.root.pretranslatedModels.get(url) === undefined
+    );
+    if (missingImports.length > 0) {
       allMissing = {...allMissing, urls: missingImports};
     }
 
@@ -358,6 +356,9 @@ class ImportsAndTablesStep implements TranslationStep {
     }
 
     for (const child of that.childTranslators.values()) {
+      if (that.root.pretranslatedModels.get(child.sourceURL)) {
+        continue;
+      }
       const kidNeeds = child.importsAndTablesStep.step(child);
       if (isNeedResponse(kidNeeds)) {
         return kidNeeds;
@@ -433,6 +434,9 @@ class ASTStep implements TranslationStep {
     // Now make sure that every child has fully translated itself
     // before this tree is ready to also translate ...
     for (const child of that.childTranslators.values()) {
+      if (that.root.pretranslatedModels.get(child.sourceURL)) {
+        continue;
+      }
       const kidNeeds = child.astStep.step(child);
       if (isNeedResponse(kidNeeds)) {
         return kidNeeds;
@@ -608,6 +612,18 @@ class TranslateStep implements TranslationStep {
       return this.response;
     }
 
+    const pretranslate = that.root.pretranslatedModels.get(that.sourceURL);
+    if (pretranslate !== undefined) {
+      that.setTranslationResults(pretranslate);
+      return {
+        // TODO handle extendingModel
+        translated: pretranslate,
+        final: true,
+        // TODO
+        fromSources: [],
+      };
+    }
+
     // begin with the compiler flags of the model we are extending
     if (extendingModel && !this.importedAnnotations) {
       const tagParse = Tag.annotationToTag(extendingModel.annotation, {
@@ -635,9 +651,7 @@ class TranslateStep implements TranslationStep {
           if (docCompile.needs) {
             return docCompile.needs;
           } else {
-            that.modelDef = docCompile.modelDef;
-            that.queryList = docCompile.queryList;
-            that.sqlBlocks = docCompile.sqlBlocks;
+            that.setTranslationResults(docCompile);
             break;
           }
         }
@@ -674,7 +688,6 @@ class TranslateStep implements TranslationStep {
 export abstract class MalloyTranslation {
   abstract root: MalloyTranslator;
   childTranslators: Map<string, MalloyTranslation>;
-  urlIsFullPath?: boolean;
   queryList: Query[] = [];
   sqlBlocks: SQLSourceDef[] = [];
   modelDef: ModelDef;
@@ -723,6 +736,30 @@ export abstract class MalloyTranslation {
     this.references = new ReferenceList(sourceURL);
   }
 
+  _urlIsFullPath: boolean | undefined = undefined;
+  get urlIsFullPath(): boolean {
+    if (this._urlIsFullPath === undefined) {
+      try {
+        const _checkFull = new URL(this.sourceURL);
+        this._urlIsFullPath = true;
+      } catch (e) {
+        const msg = e instanceof Error ? e.message : '';
+        this._urlIsFullPath = false;
+        this.root.logError(
+          'failed-to-compute-absolute-import-url',
+          `Could not compute full path URL: ${msg}`
+        );
+      }
+    }
+    return this._urlIsFullPath;
+  }
+
+  public setTranslationResults(documentDef: DocumentDef) {
+    this.modelDef = documentDef.modelDef;
+    this.queryList = documentDef.queryList;
+    this.sqlBlocks = documentDef.sqlBlocks;
+  }
+
   addChild(url: string): void {
     if (!this.childTranslators.get(url)) {
       this.childTranslators.set(url, new MalloyChildTranslator(url, this.root));
@@ -1025,6 +1062,7 @@ export class MalloyChildTranslator extends MalloyTranslation {
 export class MalloyTranslator extends MalloyTranslation {
   schemaZone = new Zone<SourceDef>();
   importZone = new Zone<string>();
+  pretranslatedModels = new Map<string, DocumentDef>();
   sqlQueryZone = new Zone<SQLSourceDef>();
   logger: BaseMessageLogger;
   readonly root: MalloyTranslator;
@@ -1046,6 +1084,9 @@ export class MalloyTranslator extends MalloyTranslation {
     this.schemaZone.updateFrom(dd.tables, dd.errors?.tables);
     this.importZone.updateFrom(dd.urls, dd.errors?.urls);
     this.sqlQueryZone.updateFrom(dd.compileSQL, dd.errors?.compileSQL);
+    for (const url in dd.translations) {
+      this.pretranslatedModels.set(url, dd.translations[url]);
+    }
   }
 
   logError<T extends MessageCode>(
@@ -1064,18 +1105,22 @@ interface ErrorData {
   tables: Record<string, string>;
   urls: Record<string, string>;
   compileSQL: Record<string, string>;
+  translations: Record<string, string>;
 }
 
 export interface URLData {
   urls: ZoneData<string>;
 }
+export interface ModelData {
+  translations: ZoneData<DocumentDef>;
+}
 export interface SchemaData {
   tables: ZoneData<SourceDef>;
 }
 export interface SQLSources {
   compileSQL: ZoneData<SQLSourceDef>;
 }
-export interface UpdateData extends URLData, SchemaData, SQLSources {
+export interface UpdateData extends URLData, SchemaData, SQLSources, ModelData {
   errors: Partial<ErrorData>;
 }
 export type ParseUpdate = Partial<UpdateData>;

packages/malloy/src/lang/test/pretranslate.spec.ts

@@ -0,0 +1,43 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+import './parse-expects';
+import {TestTranslator} from './test-translator';
+
+describe('pretranslated models', () => {
+  test('import of pretranslated', () => {
+    const docParse = new TestTranslator('import "child"');
+    const xr = docParse.unresolved();
+    expect(docParse).toParse();
+    expect(xr).toEqual({urls: ['internal://test/langtests/child']});
+    docParse.update({
+      translations: {
+        'internal://test/langtests/child': {
+          modelDef: {
+            name: 'child',
+            exports: ['foo'],
+            contents: {
+              foo: {
+                type: 'table',
+                tablePath: 'foo',
+                connection: 'duckdb',
+                dialect: 'duckdb',
+                name: 'foo',
+                fields: [],
+              },
+            },
+          },
+          queryList: [],
+          sqlBlocks: [],
+        },
+      },
+    });
+    expect(docParse).toTranslate();
+    const foo = docParse.getSourceDef('foo');
+    expect(foo).toBeDefined();
+  });
+});

packages/malloy/src/lang/translate-response.ts

@@ -23,10 +23,9 @@
 
 import {
   Annotation,
+  DocumentDef,
   ModelDef,
-  Query,
   SQLSentence,
-  SQLSourceDef,
 } from '../model/malloy_types';
 import {MalloyElement} from './ast';
 import {LogMessage} from './parse-log';
@@ -93,16 +92,13 @@ export type CompletionsResponse = Partial<Completions>;
 interface HelpContext extends NeededData, ProblemResponse, FinalResponse {
   helpContext: DocumentHelpContext | undefined;
 }
+
 export type HelpContextResponse = Partial<HelpContext>;
 interface TranslatedResponseData
   extends NeededData,
     ProblemResponse,
     FinalResponse {
-  translated: {
-    modelDef: ModelDef;
-    queryList: Query[];
-    sqlBlocks: SQLSourceDef[];
-  };
+  translated: DocumentDef;
   fromSources: string[];
 }