-
Notifications
You must be signed in to change notification settings - Fork 9
Open
Labels
bugSomething isn't workingSomething isn't working
Description
https://cloud.onehouse.ai/c3eb3868-6979-41cd-9018-952d29a43337/data/lakes/iceberg/databases/taxis
root@spark:/opt/lakeview# cat iceberg.yaml
version: V1
onehouseClientConfig:
# can be obtained from the Onehouse console
projectId: c3eb3868-6979-41cd-9018-952d29a43337
apiKey: asU2Pb3XaNAc4JwkkWpNUQ==
apiSecret: IBaLVxloIzU36heBooOBsPp5MhD6ijjyIk88zvH2ggs=
userId: x2gblCN8xNSurvCsqDaGJ84zy913
fileSystemConfiguration:
# Provide either s3Config or gcsConfig
s3Config:
region: us-east-1
accessKey: admin
accessSecret: password
endpoint: http://minio:9000
metadataExtractorConfig:
jobRunMode: ONCE
pathExclusionPatterns:
parserConfig:
- lake: delta
databases:
- name: people
basePaths: ["s3://warehouse/people"]
- lake: iceberg
databases:
- name: taxis
basePaths: ["s3://warehouse/taxis"]
# Add additional lakes and databases as needed
root@spark:/opt/lakeview# java -jar LakeView-1.0-SNAPSHOT-all.jar -p '/opt/lakeview/iceberg.yaml'
17:11:51.495 [main] INFO com.onehouse.Main - Starting LakeView extractor service
17:11:51.644 [main] INFO com.onehouse.RuntimeModule - Spinning up 70 threads
17:11:51.985 [main] INFO com.onehouse.metrics.MetricsServer - Starting metrics server
17:11:51.996 [main] INFO c.o.m.TableDiscoveryAndUploadJob - Running metadata-extractor one time
17:11:51.996 [main] INFO c.o.m.TableDiscoveryService - Starting table discover service, excluding []
17:11:51.997 [main] INFO c.o.m.TableDiscoveryService - Discovering tables in s3://warehouse/people
17:11:52.071 [main] INFO c.o.m.TableDiscoveryService - Discovering tables in s3://warehouse/taxis
17:11:52.187 [metadata-extractor-3] INFO c.o.m.TableMetadataUploaderService - Uploading metadata of following tables: [Table(absoluteTableUri=s3://warehouse/taxis, databaseName=taxis, lakeName=iceberg, tableId=null)]
17:11:52.189 [metadata-extractor-3] INFO c.o.m.TableMetadataUploaderService - Fetching checkpoint for tables: [Table(absoluteTableUri=s3://warehouse/taxis, databaseName=taxis, lakeName=iceberg, tableId=1fbfcc05-408b-3e43-b95e-af8bb335e97d)]
17:11:52.471 [metadata-extractor-3] INFO c.o.m.TableMetadataUploaderService - Initializing following tables [Table(absoluteTableUri=s3://warehouse/taxis, databaseName=taxis, lakeName=iceberg, tableId=1fbfcc05-408b-3e43-b95e-af8bb335e97d)]
17:11:52.784 [metadata-extractor-4] INFO c.o.m.TimelineCommitInstantsUploader - uploading instants in table: Table(absoluteTableUri=s3://warehouse/taxis, databaseName=taxis, lakeName=iceberg, tableId=1fbfcc05-408b-3e43-b95e-af8bb335e97d) timeline: COMMIT_TIMELINE_TYPE_ARCHIVED
17:11:52.803 [metadata-extractor-2] INFO c.o.m.TimelineCommitInstantsUploader - Processing 1 instants in table Table(absoluteTableUri=s3://warehouse/taxis, databaseName=taxis, lakeName=iceberg, tableId=1fbfcc05-408b-3e43-b95e-af8bb335e97d) timeline COMMIT_TIMELINE_TYPE_ARCHIVED sequentially in 1 batches
17:11:52.803 [metadata-extractor-2] INFO c.o.m.TimelineCommitInstantsUploader - uploading batch 1 for table Table(absoluteTableUri=s3://warehouse/taxis, databaseName=taxis, lakeName=iceberg, tableId=1fbfcc05-408b-3e43-b95e-af8bb335e97d) timeline: COMMIT_TIMELINE_TYPE_ARCHIVED
17:11:53.185 [metadata-extractor-2] INFO c.o.m.TimelineCommitInstantsUploader - uploading instants in table: Table(absoluteTableUri=s3://warehouse/taxis, databaseName=taxis, lakeName=iceberg, tableId=1fbfcc05-408b-3e43-b95e-af8bb335e97d) timeline: COMMIT_TIMELINE_TYPE_ACTIVE
17:11:53.203 [metadata-extractor-3] INFO c.o.m.TimelineCommitInstantsUploader - Processing 3 instants in table Table(absoluteTableUri=s3://warehouse/taxis, databaseName=taxis, lakeName=iceberg, tableId=1fbfcc05-408b-3e43-b95e-af8bb335e97d) timeline COMMIT_TIMELINE_TYPE_ACTIVE sequentially in 1 batches
17:11:53.204 [metadata-extractor-2] INFO c.o.m.TimelineCommitInstantsUploader - uploading batch 2 for table Table(absoluteTableUri=s3://warehouse/taxis, databaseName=taxis, lakeName=iceberg, tableId=1fbfcc05-408b-3e43-b95e-af8bb335e97d) timeline: COMMIT_TIMELINE_TYPE_ACTIVE
17:11:53.571 [metadata-extractor-2] INFO c.o.m.TimelineCommitInstantsUploader - Reached end of instants in COMMIT_TIMELINE_TYPE_ACTIVE for table Table(absoluteTableUri=s3://warehouse/taxis, databaseName=taxis, lakeName=iceberg, tableId=1fbfcc05-408b-3e43-b95e-af8bb335e97d)
17:11:53.573 [main] INFO c.o.m.TableDiscoveryAndUploadJob - Run Completed
17:11:53.574 [main] INFO com.onehouse.metrics.MetricsServer - Shutting down metrics server
Reactions are currently unavailable
Metadata
Metadata
Assignees
Labels
bugSomething isn't workingSomething isn't working
