Phil/evolve backfill (#932)

* Update wording to match changes in evolutions handler

Updates the wording around evolution actions to reflect the new behavior of the
evolutions handler, which increments the backfill counter instead of re-naming
resources.

* Allow evolutions to work when publishing materializaions

The ResourceConfig Store had some baked in assumptions that it would be
operating only on captures. But now it's possible to trigger an evolution when
only publishing a materialization, because inferred schema changes can cause
incompatibilities. This updates the ResourceConfig Store to handle both capture
and materialization specs.

* allow auto-discover options to be set independently

This allows toggling the "Breaking changes re-version collections" and
"Automatically add new collections" properties indepently of one another. Both
of those toggles still require "Keep schemas up to date" to be enabled in order
to toggle them on. But now you're able to toggle on "Breaking changes
re-version collections" without also having to enable "Automatically add new
collections". As of this commit, you can now create a capture with:
`autoDiscover: { evolveIncompatibleCollections: true, addNewBindings: false }`

---------

Co-authored-by: Travis Jenkins <travis@estuary.dev>

src/api/evolutions.ts

@@ -38,6 +38,10 @@ export function toEvolutionRequest(
     const req: EvolutionRequest = { current_name: ic.collection };
     if (hasLength(ic.requires_recreation)) {
         req.new_name = suggestedName(ic.collection);
+    } else if (ic.affected_materializations) {
+        // since we're _not_ re-creating the collection, restrict the evolution to only apply to
+        // the materializations that were affected.
+        req.materializations = ic.affected_materializations.map((m) => m.name);
     }
     return req;
 }
@@ -59,6 +63,10 @@ export interface EvolutionRequest {
     // If the desired action is to re-create the collection, then this field should be set to the new name.
     // Otherwise, the evolution will only update materialization bindings to materialize into new resources.
     new_name?: string;
+    // If specified, restrict the evolution to only the given materializations.
+    // At most one of `new_name` or `materializations` may be specified, since
+    // re-creating the collection must apply to all materializations.
+    materializations?: string[];
 }
 
 export const createEvolution = (

src/lang/en-US.ts

@@ -1251,15 +1251,12 @@ const EntityEvolution: ResolvedIntlConfig['messages'] = {
     'entityEvolution.error.note': `Note: This may result in additional cost as new versions are backfilled.`,
 
     // Single quotes are special and must be doubled: https://formatjs.io/docs/core-concepts/icu-syntax#quoting--escaping
-    'entityEvolution.action.recreateOneBinding.description': `the materialization ''{materializationName}'' will be updated to materialize the collection into a new resource`,
+    'entityEvolution.action.recreateOneBinding.description': `the materialization ''{materializationName}'' will be updated to increment the backfill counter and re-materialize the collection`,
     'entityEvolution.action.recreateBindings.description': `{materializationCount} {materializationCount, plural,
         one {Materialization}
         other {Materializations}
-    } will be updated to materialize the collection into new resources`,
-    'entityEvolution.action.recreateBindings.help': `Any materializations of this collection will be updated to materialize it
-    into a new resource (database table, for example) with an incremented version suffix (like "_v2"). The collection itself will
-    have the schema updated in place, and will retain all current data. The materialization will backfill from the beginning of
-    this collection, but other bindings in the materialization will not be affected.`,
+    } will be updated to increment the backfill counters and re-materialize the collection`,
+    'entityEvolution.action.recreateBindings.help': `The materialization will be updated to increment the ''backfill'' property of the affected binding, which causes it to re-create destination resources (such as tables) and re-materialize the source collection from the beginning. Other bindings in the materialization will not be affected. The source collection will retain all current data.`,
 
     'entityEvolution.action.recreateCollection.description': `Collection will be re-created as ''{newName}'' because {reason}`,
     'entityEvolution.action.recreateCollection.help': `This will create a new collection with the name shown.

src/stores/ResourceConfig/Store.ts

@@ -34,6 +34,26 @@ import { ResourceConfigDictionary, ResourceConfigState } from './types';
 
 const STORE_KEY = 'Resource Config';
 
+// Returns the collection name for either a capture or materialization binding,
+// or throws an exception if no collection name is found.
+const getBoundCollectionName = (binding: any) => {
+    if (typeof binding.target === 'string') {
+        return binding.target;
+    }
+    if (typeof binding.source === 'string') {
+        return binding.source;
+    }
+    // This form is used for materializations and derivations that read from
+    // specific collection partitions
+    if (
+        typeof binding.source === 'object' &&
+        typeof binding.source.name === 'string'
+    ) {
+        return binding.source.name;
+    }
+    throw new Error(`no collection name found in binding: ${binding}`);
+};
+
 const populateCollections = (
     state: ResourceConfigState,
     collections: string[]
@@ -372,7 +392,7 @@ const getInitialState = (
                 const discoveredCollections: string[] = [];
 
                 value.spec.bindings.forEach((binding: any) => {
-                    discoveredCollections.push(binding.target);
+                    discoveredCollections.push(getBoundCollectionName(binding));
                 });
 
                 state.discoveredCollections = discoveredCollections;
@@ -729,13 +749,14 @@ const getInitialState = (
                 const modifiedResourceConfig: ResourceConfigDictionary = {};
 
                 sortBindings(updatedBindings).forEach((binding: any) => {
+                    const collectionName = getBoundCollectionName(binding);
                     if (
-                        !existingCollections.includes(binding.target) &&
+                        !existingCollections.includes(collectionName) &&
                         !restrictedDiscoveredCollections.includes(
-                            binding.target
+                            collectionName
                         )
                     ) {
-                        collectionsToAdd.push(binding.target);
+                        collectionsToAdd.push(collectionName);
 
                         // Keep in sync with prefillResourceConfig
                         const [name, configVal] = getResourceConfig(binding);

src/stores/SchemaEvolution/Store.ts

@@ -58,11 +58,7 @@ const getInitialState = (
     setAddNewBindings: (value, options) => {
         set(
             produce((state: SchemaEvolutionState) => {
-                const {
-                    autoDiscover,
-                    evolveIncompatibleCollections,
-                    settingsActive,
-                } = get();
+                const { autoDiscover, settingsActive } = get();
 
                 if (!settingsActive && !options?.initOnly) {
                     state.settingsActive = true;
@@ -73,11 +69,6 @@ const getInitialState = (
                     state.autoDiscover = true;
                 }
 
-                // Disable the incompatible collection evolution option when the add new bindings option is disabled.
-                if (!value && evolveIncompatibleCollections) {
-                    state.evolveIncompatibleCollections = false;
-                }
-
                 state.addNewBindings = value;
             }),
             false,
@@ -88,16 +79,15 @@ const getInitialState = (
     setEvolveIncompatibleCollections: (value, options) => {
         set(
             produce((state: SchemaEvolutionState) => {
-                const { addNewBindings, autoDiscover, settingsActive } = get();
+                const { autoDiscover, settingsActive } = get();
 
                 if (!settingsActive && !options?.initOnly) {
                     state.settingsActive = true;
                 }
 
-                // Enable auto-discovery and the add new bindings option when the incompatible collection evolution option is enabled.
-                if (value && (!autoDiscover || !addNewBindings)) {
+                // Enable auto-discovery when the incompatible collection evolution option is enabled.
+                if (value && !autoDiscover) {
                     state.autoDiscover = true;
-                    state.addNewBindings = true;
                 }
 
                 state.evolveIncompatibleCollections = value;