diff --git a/app-config.local.yaml.example b/app-config.local.yaml.example index 0d824ce8..35b7fe2c 100644 --- a/app-config.local.yaml.example +++ b/app-config.local.yaml.example @@ -35,6 +35,12 @@ backend: openchoreo: baseUrl: http://api.openchoreo.localhost:8080/api/v1 # token: "" # Optional: uncomment if you need API authentication + # OPTIONAL: For large-scale local testing, enable incremental ingestion + # incremental: + # burstLength: 16 # Duration of each burst of processing activity in seconds + # burstInterval: 8 # Interval between bursts of processing activity in seconds + # chunkSize: 512 # Number of items to fetch per API request + # restLength: 60 # Duration of rest periods between bursts in minutes # OAuth2 Client Credentials for background tasks (Catalog Entity Provider) # Required for the Catalog Provider to fetch organizations, projects, and components diff --git a/app-config.production.yaml b/app-config.production.yaml index b80cf76e..4812ad6f 100644 --- a/app-config.production.yaml +++ b/app-config.production.yaml @@ -115,9 +115,18 @@ openchoreo: # scopes: ['openid'] # Optional: uncomment to specify scopes defaultOwner: 'platformengineer' # Default owner for catalog entities + # DEFAULT: Standard scheduled ingestion (recommended for most deployments) schedule: frequency: 30 # seconds between runs (default: 30) timeout: 120 # seconds for timeout (default: 120) + # OPTIONAL: For large-scale deployments, use incremental ingestion instead + # Uncomment the section below and comment out the schedule section above + # Also update packages/backend/src/index.ts to use the incremental module + # incremental: + # burstLength: 16 # Duration of each burst of processing activity in seconds + # burstInterval: 8 # Interval between bursts of processing activity in seconds + # chunkSize: 512 # Number of items to fetch per API request + # restLength: 60 # Duration of rest periods between bursts in minutes # Feature flags for enabling/disabling OpenChoreo functionality # Environment variables: OPENCHOREO_FEATURES_WORKFLOWS_ENABLED, OPENCHOREO_FEATURES_OBSERVABILITY_ENABLED, OPENCHOREO_FEATURES_AUTH_ENABLED diff --git a/app-config.yaml b/app-config.yaml index 5da8b011..1640d93a 100644 --- a/app-config.yaml +++ b/app-config.yaml @@ -131,9 +131,19 @@ openchoreo: # scopes: ['openid'] # Optional: uncomment to specify scopes defaultOwner: 'platformengineer' # Default owner for catalog entities + + # DEFAULT: Standard scheduled ingestion (recommended for most deployments) schedule: frequency: 30 # seconds between runs (default: 30) timeout: 120 # seconds for timeout (default: 120) + # OPTIONAL: For large-scale deployments, use incremental ingestion instead + # Uncomment the section below and comment out the schedule section above + # Also update packages/backend/src/index.ts to use the incremental module + # incremental: + # burstLength: 16 # Duration of each burst of processing activity in seconds + # burstInterval: 8 # Interval between bursts of processing activity in seconds + # chunkSize: 512 # Number of items to fetch per API request + # restLength: 60 # Duration of rest periods between bursts in minutes # Feature flags for enabling/disabling OpenChoreo functionality # These can be controlled via Helm values: backstage.features.* diff --git a/packages/app/src/scaffolder/TraitsField/TraitsFieldExtension.tsx b/packages/app/src/scaffolder/TraitsField/TraitsFieldExtension.tsx index f608e62e..fb660318 100644 --- a/packages/app/src/scaffolder/TraitsField/TraitsFieldExtension.tsx +++ b/packages/app/src/scaffolder/TraitsField/TraitsFieldExtension.tsx @@ -73,8 +73,13 @@ export const TraitsField = ({ const [addedTraits, setAddedTraits] = useState(formData || []); const [selectedTrait, setSelectedTrait] = useState(''); const [loadingTraits, setLoadingTraits] = useState(false); + const [loadingMoreTraits, setLoadingMoreTraits] = useState(false); const [loadingSchema, setLoadingSchema] = useState(false); const [error, setError] = useState(null); + const [hasMoreTraits, setHasMoreTraits] = useState(true); + const [continueToken, setContinueToken] = useState( + undefined, + ); const discoveryApi = useApi(discoveryApiRef); const fetchApi = useApi(fetchApiRef); @@ -89,12 +94,16 @@ export const TraitsField = ({ useEffect(() => { let ignore = false; - const fetchTraits = async () => { + const fetchTraits = async (cursor?: string, append = false) => { if (!organizationName) { return; } - setLoadingTraits(true); + if (!append) { + setLoadingTraits(true); + } else { + setLoadingMoreTraits(true); + } setError(null); try { @@ -108,12 +117,17 @@ export const TraitsField = ({ const orgName = extractOrgName(organizationName); + // Build URL with pagination parameters + const url = new URL(`${baseUrl}/traits`); + url.searchParams.set('organizationName', orgName); + url.searchParams.set('limit', '100'); // Reasonable page size for UI + + if (cursor) { + url.searchParams.set('continue', cursor); + } + // Use fetchApi which automatically injects Backstage + IDP tokens - const response = await fetchApi.fetch( - `${baseUrl}/traits?organizationName=${encodeURIComponent( - orgName, - )}&page=1&pageSize=100`, - ); + const response = await fetchApi.fetch(url.toString()); if (!response.ok) { throw new Error(`HTTP ${response.status}: ${response.statusText}`); @@ -122,7 +136,18 @@ export const TraitsField = ({ const result = await response.json(); if (!ignore && result.success) { - setAvailableTraits(result.data.items); + const newTraits = result.data.items || []; + const metadata = result.data.metadata; + + if (append) { + setAvailableTraits(prev => [...prev, ...newTraits]); + } else { + setAvailableTraits(newTraits); + } + + // Update pagination state + setHasMoreTraits(metadata?.hasMore === true); + setContinueToken(metadata?.continue); } } catch (err) { if (!ignore) { @@ -131,10 +156,16 @@ export const TraitsField = ({ } finally { if (!ignore) { setLoadingTraits(false); + setLoadingMoreTraits(false); } } }; + // Reset pagination state when organization changes + setAvailableTraits([]); + setHasMoreTraits(true); + setContinueToken(undefined); + fetchTraits(); return () => { @@ -142,6 +173,54 @@ export const TraitsField = ({ }; }, [organizationName, discoveryApi, fetchApi]); + // Load more traits + const handleLoadMoreTraits = () => { + if (continueToken && !loadingMoreTraits) { + // We need to recreate the fetchTraits function here since it's defined in useEffect + const loadMore = async () => { + if (!organizationName) return; + + setLoadingMoreTraits(true); + setError(null); + + try { + const baseUrl = await discoveryApi.getBaseUrl('openchoreo'); + const extractOrgName = (fullOrgName: string): string => { + const parts = fullOrgName.split('/'); + return parts[parts.length - 1]; + }; + const orgName = extractOrgName(organizationName); + + const url = new URL(`${baseUrl}/traits`); + url.searchParams.set('organizationName', orgName); + url.searchParams.set('limit', '100'); + url.searchParams.set('continue', continueToken); + + const response = await fetchApi.fetch(url.toString()); + if (!response.ok) { + throw new Error(`HTTP ${response.status}: ${response.statusText}`); + } + + const result = await response.json(); + if (result.success) { + const newTraits = result.data.items || []; + const metadata = result.data.metadata; + + setAvailableTraits(prev => [...prev, ...newTraits]); + setHasMoreTraits(metadata?.hasMore === true); + setContinueToken(metadata?.continue); + } + } catch (err) { + setError(`Failed to load more traits: ${err}`); + } finally { + setLoadingMoreTraits(false); + } + }; + + loadMore(); + } + }; + // Fetch schema for selected trait and add it const handleAddTrait = async () => { if (!selectedTrait || !organizationName) { @@ -273,6 +352,24 @@ export const TraitsField = ({ {trait.name} ))} + + {/* Load More Button */} + {!loadingTraits && hasMoreTraits && ( + + {loadingMoreTraits ? ( + <> + + Loading more traits... + + ) : ( + 'Load more traits...' + )} + + )}