feat: Add Incremental Entity Provider and Global API Pagination Support

app-config.local.yaml.example

@@ -35,6 +35,12 @@ backend:
 openchoreo:
   baseUrl: http://api.openchoreo.localhost:8080/api/v1
   # token: ""  # Optional: uncomment if you need API authentication
+  # OPTIONAL: For large-scale local testing, enable incremental ingestion
+  # incremental:
+  #   burstLength: 16 # Duration of each burst of processing activity in seconds
+  #   burstInterval: 8 # Interval between bursts of processing activity in seconds
+  #   chunkSize: 512 # Number of items to fetch per API request
+  #   restLength: 60 # Duration of rest periods between bursts in minutes
 
   # OAuth2 Client Credentials for background tasks (Catalog Entity Provider)
   # Required for the Catalog Provider to fetch organizations, projects, and components

app-config.production.yaml

@@ -115,9 +115,18 @@ openchoreo:
     # scopes: ['openid'] # Optional: uncomment to specify scopes
 
   defaultOwner: 'platformengineer' # Default owner for catalog entities
+  # DEFAULT: Standard scheduled ingestion (recommended for most deployments)
   schedule:
     frequency: 30 # seconds between runs (default: 30)
     timeout: 120 # seconds for timeout (default: 120)
+  # OPTIONAL: For large-scale deployments, use incremental ingestion instead
+  # Uncomment the section below and comment out the schedule section above
+  # Also update packages/backend/src/index.ts to use the incremental module
+  # incremental:
+  #   burstLength: 16 # Duration of each burst of processing activity in seconds
+  #   burstInterval: 8 # Interval between bursts of processing activity in seconds
+  #   chunkSize: 512 # Number of items to fetch per API request
+  #   restLength: 60 # Duration of rest periods between bursts in minutes
 
   # Feature flags for enabling/disabling OpenChoreo functionality
   # Environment variables: OPENCHOREO_FEATURES_WORKFLOWS_ENABLED, OPENCHOREO_FEATURES_OBSERVABILITY_ENABLED, OPENCHOREO_FEATURES_AUTH_ENABLED

app-config.yaml

@@ -131,9 +131,19 @@ openchoreo:
     # scopes: ['openid'] # Optional: uncomment to specify scopes
 
   defaultOwner: 'platformengineer' # Default owner for catalog entities
+
+  # DEFAULT: Standard scheduled ingestion (recommended for most deployments)
   schedule:
     frequency: 30 # seconds between runs (default: 30)
     timeout: 120 # seconds for timeout (default: 120)
+  # OPTIONAL: For large-scale deployments, use incremental ingestion instead
+  # Uncomment the section below and comment out the schedule section above
+  # Also update packages/backend/src/index.ts to use the incremental module
+  # incremental:
+  #   burstLength: 16 # Duration of each burst of processing activity in seconds
+  #   burstInterval: 8 # Interval between bursts of processing activity in seconds
+  #   chunkSize: 512 # Number of items to fetch per API request
+  #   restLength: 60 # Duration of rest periods between bursts in minutes
 
   # Feature flags for enabling/disabling OpenChoreo functionality
   # These can be controlled via Helm values: backstage.features.*

packages/app/src/scaffolder/TraitsField/TraitsFieldExtension.tsx

@@ -73,8 +73,13 @@ export const TraitsField = ({
   const [addedTraits, setAddedTraits] = useState<AddedTrait[]>(formData || []);
   const [selectedTrait, setSelectedTrait] = useState<string>('');
   const [loadingTraits, setLoadingTraits] = useState(false);
+  const [loadingMoreTraits, setLoadingMoreTraits] = useState(false);
   const [loadingSchema, setLoadingSchema] = useState(false);
   const [error, setError] = useState<string | null>(null);
+  const [hasMoreTraits, setHasMoreTraits] = useState(true);
+  const [continueToken, setContinueToken] = useState<string | undefined>(
+    undefined,
+  );
 
   const discoveryApi = useApi(discoveryApiRef);
   const fetchApi = useApi(fetchApiRef);
@@ -89,12 +94,16 @@ export const TraitsField = ({
   useEffect(() => {
     let ignore = false;
 
-    const fetchTraits = async () => {
+    const fetchTraits = async (cursor?: string, append = false) => {
       if (!organizationName) {
         return;
       }
 
-      setLoadingTraits(true);
+      if (!append) {
+        setLoadingTraits(true);
+      } else {
+        setLoadingMoreTraits(true);
+      }
       setError(null);
 
       try {
@@ -108,12 +117,17 @@ export const TraitsField = ({
 
         const orgName = extractOrgName(organizationName);
 
+        // Build URL with pagination parameters
+        const url = new URL(`${baseUrl}/traits`);
+        url.searchParams.set('organizationName', orgName);
+        url.searchParams.set('limit', '100'); // Reasonable page size for UI
+
+        if (cursor) {
+          url.searchParams.set('continue', cursor);
+        }
+
         // Use fetchApi which automatically injects Backstage + IDP tokens
-        const response = await fetchApi.fetch(
-          `${baseUrl}/traits?organizationName=${encodeURIComponent(
-            orgName,
-          )}&page=1&pageSize=100`,
-        );
+        const response = await fetchApi.fetch(url.toString());
 
         if (!response.ok) {
           throw new Error(`HTTP ${response.status}: ${response.statusText}`);
@@ -122,7 +136,18 @@ export const TraitsField = ({
         const result = await response.json();
 
         if (!ignore && result.success) {
-          setAvailableTraits(result.data.items);
+          const newTraits = result.data.items || [];
+          const metadata = result.data.metadata;
+
+          if (append) {
+            setAvailableTraits(prev => [...prev, ...newTraits]);
+          } else {
+            setAvailableTraits(newTraits);
+          }
+
+          // Update pagination state
+          setHasMoreTraits(metadata?.hasMore === true);
+          setContinueToken(metadata?.continue);
         }
       } catch (err) {
         if (!ignore) {
@@ -131,17 +156,71 @@ export const TraitsField = ({
       } finally {
         if (!ignore) {
           setLoadingTraits(false);
+          setLoadingMoreTraits(false);
         }
       }
     };
 
+    // Reset pagination state when organization changes
+    setAvailableTraits([]);
+    setHasMoreTraits(true);
+    setContinueToken(undefined);
+
     fetchTraits();
 
     return () => {
       ignore = true;
     };
   }, [organizationName, discoveryApi, fetchApi]);
 
+  // Load more traits
+  const handleLoadMoreTraits = () => {
+    if (continueToken && !loadingMoreTraits) {
+      // We need to recreate the fetchTraits function here since it's defined in useEffect
+      const loadMore = async () => {
+        if (!organizationName) return;
+
+        setLoadingMoreTraits(true);
+        setError(null);
+
+        try {
+          const baseUrl = await discoveryApi.getBaseUrl('openchoreo');
+          const extractOrgName = (fullOrgName: string): string => {
+            const parts = fullOrgName.split('/');
+            return parts[parts.length - 1];
+          };
+          const orgName = extractOrgName(organizationName);
+
+          const url = new URL(`${baseUrl}/traits`);
+          url.searchParams.set('organizationName', orgName);
+          url.searchParams.set('limit', '100');
+          url.searchParams.set('continue', continueToken);
+
+          const response = await fetchApi.fetch(url.toString());
+          if (!response.ok) {
+            throw new Error(`HTTP ${response.status}: ${response.statusText}`);
+          }
+
+          const result = await response.json();
+          if (result.success) {
+            const newTraits = result.data.items || [];
+            const metadata = result.data.metadata;
+
+            setAvailableTraits(prev => [...prev, ...newTraits]);
+            setHasMoreTraits(metadata?.hasMore === true);
+            setContinueToken(metadata?.continue);
+          }
+        } catch (err) {
+          setError(`Failed to load more traits: ${err}`);
+        } finally {
+          setLoadingMoreTraits(false);
+        }
+      };
+
+      loadMore();
+    }
+  };
+
   // Fetch schema for selected trait and add it
   const handleAddTrait = async () => {
     if (!selectedTrait || !organizationName) {
@@ -273,6 +352,24 @@ export const TraitsField = ({
                     {trait.name}
                   </MenuItem>
                 ))}
+
+              {/* Load More Button */}
+              {!loadingTraits && hasMoreTraits && (
+                <MenuItem
+                  disabled={loadingMoreTraits}
+                  onClick={handleLoadMoreTraits}
+                  style={{ justifyContent: 'center', fontStyle: 'italic' }}
+                >
+                  {loadingMoreTraits ? (
+                    <>
+                      <CircularProgress size={20} style={{ marginRight: 8 }} />
+                      Loading more traits...
+                    </>
+                  ) : (
+                    'Load more traits...'
+                  )}
+                </MenuItem>
+              )}
             </Select>
           </FormControl>
           <Button

packages/backend/package.json

@@ -50,6 +50,7 @@
     "@openchoreo/backstage-plugin-platform-engineer-core-backend": "workspace:^",
     "@openchoreo/backstage-plugin-scaffolder-backend-module": "workspace:^",
     "@openchoreo/openchoreo-auth": "workspace:^",
+    "@openchoreo/plugin-catalog-backend-module-openchoreo-incremental": "workspace:^",
     "app": "link:../app",
     "better-sqlite3": "9.6.0",
     "cookie-parser": "1.4.7",

packages/backend/src/index.ts

@@ -12,6 +12,16 @@ import { rootHttpRouterServiceFactory } from '@backstage/backend-defaults/rootHt
 import { immediateCatalogServiceFactory } from '@openchoreo/backstage-plugin-catalog-backend-module';
 import { createIdpTokenHeaderMiddleware } from '@openchoreo/openchoreo-auth';
 
+/**
+ * OPTIONAL: For large-scale deployments, use the incremental ingestion module
+ *
+ * ----------------------------------------------------------------------
+ * INCREMENTAL INGESTION: STEP 1 of 2
+ * ----------------------------------------------------------------------
+ */
+// UNCOMMENT this import line below.
+// import { catalogModuleOpenchoreoIncrementalProvider } from '@openchoreo/plugin-catalog-backend-module-openchoreo-incremental';
+
 const backend = createBackend();
 
 // Configure root HTTP router with IDP token header middleware
@@ -60,6 +70,21 @@ backend.add(
   import('@backstage/plugin-catalog-backend-module-scaffolder-entity-model'),
 );
 
+/**
+ * ----------------------------------------------------------------------
+ * INCREMENTAL INGESTION: STEP 2 of 2
+ * ----------------------------------------------------------------------
+ *
+ *  Note: You must also update app-config.yaml to use:
+ * 'openchoreo.incremental' instead of 'openchoreo.schedule'
+ */
+
+// UNCOMMENT the block below..
+// backend.add(
+//   import('@openchoreo/plugin-catalog-backend-module-openchoreo-incremental'),
+// );
+// backend.add(catalogModuleOpenchoreoIncrementalProvider);
+
 // See https://backstage.io/docs/features/software-catalog/configuration#subscribing-to-catalog-errors
 backend.add(import('@backstage/plugin-catalog-backend-module-logs'));