-
Notifications
You must be signed in to change notification settings - Fork 28
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add devcontainers support to the repository (#267)
### Description This PR aims to introduce devcontainer support to this repository. With this addition, developers will be able to quickly set up and run a dockerized environment in VSCode, GitHub Codespaces, or any other platform that supports devcontainers. ### Changelog - [x] Add devcontainers support to the repository.
- Loading branch information
1 parent
261b1bd
commit 512894b
Showing
3 changed files
with
164 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,30 @@ | ||
// For format details, see https://aka.ms/devcontainer.json. For config options, see the | ||
// README at: https://github.com/devcontainers/templates/tree/main/src/python | ||
{ | ||
"name": "dbt-dremio-devcontainer", | ||
// Use a Docker Compose file | ||
"dockerComposeFile": "../.docker/docker-compose.yml", | ||
"service": "dev", | ||
"workspaceFolder": "/usr/src/app", | ||
|
||
"runArgs": [ | ||
"--userns=keep-id:uid=1000,gid=1000" | ||
], | ||
"containerUser": "vscode", | ||
"updateRemoteUserUID": true, | ||
|
||
// Configure tool-specific properties. | ||
"customizations": { | ||
"vscode": { | ||
"extensions": [ | ||
"ms-python.python", | ||
"ms-python.vscode-pylance", | ||
"ms-azuretools.vscode-docker", | ||
"eamodio.gitlens" | ||
], | ||
"settings": { | ||
"terminal.integrated.shell.linux": "/bin/bash" | ||
} | ||
} | ||
} | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,101 @@ | ||
services: | ||
# Nessie Catalog Server Using In-Memory Store | ||
nessie: | ||
image: projectnessie/nessie:latest | ||
container_name: catalog | ||
networks: | ||
- dremio-laptop-lakehouse | ||
ports: | ||
- 19120:19120 | ||
|
||
# Minio Storage Server | ||
minio: | ||
image: minio/minio:latest | ||
container_name: storage | ||
environment: | ||
- MINIO_ROOT_USER=admin | ||
- MINIO_ROOT_PASSWORD=password | ||
- MINIO_DOMAIN=storage | ||
- MINIO_REGION_NAME=us-east-1 | ||
- MINIO_REGION=us-east-1 | ||
networks: | ||
- dremio-laptop-lakehouse | ||
ports: | ||
- 9001:9001 | ||
- 9000:9000 | ||
command: [ "server", "/data", "--console-address", ":9001" ] | ||
volumes: | ||
- minio_data:/data | ||
|
||
minio-setup: | ||
image: minio/mc | ||
container_name: minio-setup | ||
depends_on: | ||
- minio | ||
entrypoint: > | ||
/bin/sh -c " | ||
until (echo > /dev/tcp/minio/9000) >/dev/null 2>&1; do | ||
echo 'Waiting for MinIO...'; | ||
sleep 2; | ||
done; | ||
mc alias set myminio http://minio:9000 admin password; | ||
mc mb myminio/datalake; | ||
mc ls myminio; | ||
" | ||
networks: | ||
- dremio-laptop-lakehouse | ||
|
||
# Dremio | ||
dremio: | ||
image: dremio/dremio-oss:latest | ||
ports: | ||
- 9047:9047 | ||
- 31010:31010 | ||
- 32010:32010 | ||
- 45678:45678 | ||
container_name: dremio | ||
environment: | ||
- DREMIO_JAVA_SERVER_EXTRA_OPTS=-Dpaths.dist=file:///opt/dremio/data/dist -Ddebug.addDefaultUser=true | ||
- SERVICES_COORDINATOR_ENABLED=true | ||
- SERVICES_EXECUTOR_ENABLED=true | ||
networks: | ||
- dremio-laptop-lakehouse | ||
volumes: | ||
- dremio_data:/opt/dremio/data:rw | ||
# Workaround for permission issues in podman | ||
user: "0" | ||
|
||
dremio-setup: | ||
image: alpine:latest | ||
container_name: dremio-setup | ||
depends_on: | ||
- dremio | ||
volumes: | ||
- ./dremio-setup.sh:/dremio-setup.sh | ||
command: sh /dremio-setup.sh | ||
networks: | ||
- dremio-laptop-lakehouse | ||
|
||
# Dev environment container | ||
dev: | ||
image: mcr.microsoft.com/devcontainers/python:1-3.9-bullseye | ||
container_name: dev | ||
environment: | ||
- DBT_PROFILES_DIR=/usr/src/app/ | ||
volumes: | ||
- ..:/usr/src/app:rw | ||
- dremio_data:/mnt/dremio_data:rw | ||
working_dir: /usr/src/app | ||
command: /bin/bash -c "pip install --upgrade pip && pip install --user -r dev_requirements.txt && pip install --user . && tail -f /dev/null && /bin/bash" | ||
tty: true | ||
networks: | ||
- dremio-laptop-lakehouse | ||
depends_on: | ||
- dremio | ||
|
||
networks: | ||
dremio-laptop-lakehouse: | ||
|
||
volumes: | ||
dremio_data: | ||
minio_data: |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,33 @@ | ||
#!/bin/sh | ||
|
||
# Install required tools | ||
apk add --no-cache curl jq | ||
|
||
# Wait for Dremio to be ready | ||
until curl -s http://dremio:9047; do | ||
echo "Waiting for Dremio..." | ||
sleep 5 | ||
done | ||
|
||
echo "Dremio is up. Proceeding with configuration..." | ||
|
||
# Log in to Dremio to get the auth token | ||
AUTH_TOKEN=$(curl -s -X POST "http://dremio:9047/apiv2/login" \ | ||
-H "Content-Type: application/json" \ | ||
--data "{\"userName\":\"dremio\", \"password\":\"dremio123\"}" | jq -r .token) | ||
|
||
# Check if AUTH_TOKEN is not empty | ||
if [ -z "$AUTH_TOKEN" ]; then | ||
echo "Failed to obtain Dremio auth token" | ||
exit 1 | ||
fi | ||
|
||
echo "Obtained Dremio auth token" | ||
|
||
# Create the S3 source in Dremio | ||
curl -s -X PUT "http://dremio:9047/apiv2/source/S3Source" \ | ||
-H "Content-Type: application/json" \ | ||
-H "Authorization: _dremio$AUTH_TOKEN" \ | ||
--data "{\"name\":\"S3Source\",\"config\":{\"credentialType\":\"ACCESS_KEY\",\"accessKey\":\"admin\",\"accessSecret\":\"password\",\"secure\":false,\"externalBucketList\":[],\"enableAsync\":true,\"enableFileStatusCheck\":true,\"rootPath\":\"/\",\"defaultCtasFormat\":\"ICEBERG\",\"propertyList\":[{\"name\":\"fs.s3a.path.style.access\",\"value\":\"true\"},{\"name\":\"fs.s3a.endpoint\",\"value\":\"minio:9000\"},{\"name\":\"dremio.s3.compat\",\"value\":\"true\"}],\"whitelistedBuckets\":[],\"isCachingEnabled\":false,\"maxCacheSpacePct\":100},\"type\":\"S3\",\"metadataPolicy\":{\"deleteUnavailableDatasets\":true,\"autoPromoteDatasets\":false,\"namesRefreshMillis\":3600000,\"datasetDefinitionRefreshAfterMillis\":3600000,\"datasetDefinitionExpireAfterMillis\":10800000,\"authTTLMillis\":86400000,\"updateMode\":\"PREFETCH_QUERIED\"}}" | ||
|
||
echo "S3 Source created in Dremio" |