Skip to content

Commit

Permalink
Add devcontainers support to the repository (#267)
Browse files Browse the repository at this point in the history
### Description

This PR aims to introduce devcontainer support to this repository. With
this addition, developers will be able to quickly set up and run a
dockerized environment in VSCode, GitHub Codespaces, or any other
platform that supports devcontainers.

### Changelog

-   [x] Add devcontainers support to the repository.
  • Loading branch information
simonpannek authored Feb 14, 2025
1 parent 261b1bd commit 512894b
Show file tree
Hide file tree
Showing 3 changed files with 164 additions and 0 deletions.
30 changes: 30 additions & 0 deletions .devcontainer/devcontainer.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
// For format details, see https://aka.ms/devcontainer.json. For config options, see the
// README at: https://github.com/devcontainers/templates/tree/main/src/python
{
"name": "dbt-dremio-devcontainer",
// Use a Docker Compose file
"dockerComposeFile": "../.docker/docker-compose.yml",
"service": "dev",
"workspaceFolder": "/usr/src/app",

"runArgs": [
"--userns=keep-id:uid=1000,gid=1000"
],
"containerUser": "vscode",
"updateRemoteUserUID": true,

// Configure tool-specific properties.
"customizations": {
"vscode": {
"extensions": [
"ms-python.python",
"ms-python.vscode-pylance",
"ms-azuretools.vscode-docker",
"eamodio.gitlens"
],
"settings": {
"terminal.integrated.shell.linux": "/bin/bash"
}
}
}
}
101 changes: 101 additions & 0 deletions .docker/docker-compose.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,101 @@
services:
# Nessie Catalog Server Using In-Memory Store
nessie:
image: projectnessie/nessie:latest
container_name: catalog
networks:
- dremio-laptop-lakehouse
ports:
- 19120:19120

# Minio Storage Server
minio:
image: minio/minio:latest
container_name: storage
environment:
- MINIO_ROOT_USER=admin
- MINIO_ROOT_PASSWORD=password
- MINIO_DOMAIN=storage
- MINIO_REGION_NAME=us-east-1
- MINIO_REGION=us-east-1
networks:
- dremio-laptop-lakehouse
ports:
- 9001:9001
- 9000:9000
command: [ "server", "/data", "--console-address", ":9001" ]
volumes:
- minio_data:/data

minio-setup:
image: minio/mc
container_name: minio-setup
depends_on:
- minio
entrypoint: >
/bin/sh -c "
until (echo > /dev/tcp/minio/9000) >/dev/null 2>&1; do
echo 'Waiting for MinIO...';
sleep 2;
done;
mc alias set myminio http://minio:9000 admin password;
mc mb myminio/datalake;
mc ls myminio;
"
networks:
- dremio-laptop-lakehouse

# Dremio
dremio:
image: dremio/dremio-oss:latest
ports:
- 9047:9047
- 31010:31010
- 32010:32010
- 45678:45678
container_name: dremio
environment:
- DREMIO_JAVA_SERVER_EXTRA_OPTS=-Dpaths.dist=file:///opt/dremio/data/dist -Ddebug.addDefaultUser=true
- SERVICES_COORDINATOR_ENABLED=true
- SERVICES_EXECUTOR_ENABLED=true
networks:
- dremio-laptop-lakehouse
volumes:
- dremio_data:/opt/dremio/data:rw
# Workaround for permission issues in podman
user: "0"

dremio-setup:
image: alpine:latest
container_name: dremio-setup
depends_on:
- dremio
volumes:
- ./dremio-setup.sh:/dremio-setup.sh
command: sh /dremio-setup.sh
networks:
- dremio-laptop-lakehouse

# Dev environment container
dev:
image: mcr.microsoft.com/devcontainers/python:1-3.9-bullseye
container_name: dev
environment:
- DBT_PROFILES_DIR=/usr/src/app/
volumes:
- ..:/usr/src/app:rw
- dremio_data:/mnt/dremio_data:rw
working_dir: /usr/src/app
command: /bin/bash -c "pip install --upgrade pip && pip install --user -r dev_requirements.txt && pip install --user . && tail -f /dev/null && /bin/bash"
tty: true
networks:
- dremio-laptop-lakehouse
depends_on:
- dremio

networks:
dremio-laptop-lakehouse:

volumes:
dremio_data:
minio_data:
33 changes: 33 additions & 0 deletions .docker/dremio-setup.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
#!/bin/sh

# Install required tools
apk add --no-cache curl jq

# Wait for Dremio to be ready
until curl -s http://dremio:9047; do
echo "Waiting for Dremio..."
sleep 5
done

echo "Dremio is up. Proceeding with configuration..."

# Log in to Dremio to get the auth token
AUTH_TOKEN=$(curl -s -X POST "http://dremio:9047/apiv2/login" \
-H "Content-Type: application/json" \
--data "{\"userName\":\"dremio\", \"password\":\"dremio123\"}" | jq -r .token)

# Check if AUTH_TOKEN is not empty
if [ -z "$AUTH_TOKEN" ]; then
echo "Failed to obtain Dremio auth token"
exit 1
fi

echo "Obtained Dremio auth token"

# Create the S3 source in Dremio
curl -s -X PUT "http://dremio:9047/apiv2/source/S3Source" \
-H "Content-Type: application/json" \
-H "Authorization: _dremio$AUTH_TOKEN" \
--data "{\"name\":\"S3Source\",\"config\":{\"credentialType\":\"ACCESS_KEY\",\"accessKey\":\"admin\",\"accessSecret\":\"password\",\"secure\":false,\"externalBucketList\":[],\"enableAsync\":true,\"enableFileStatusCheck\":true,\"rootPath\":\"/\",\"defaultCtasFormat\":\"ICEBERG\",\"propertyList\":[{\"name\":\"fs.s3a.path.style.access\",\"value\":\"true\"},{\"name\":\"fs.s3a.endpoint\",\"value\":\"minio:9000\"},{\"name\":\"dremio.s3.compat\",\"value\":\"true\"}],\"whitelistedBuckets\":[],\"isCachingEnabled\":false,\"maxCacheSpacePct\":100},\"type\":\"S3\",\"metadataPolicy\":{\"deleteUnavailableDatasets\":true,\"autoPromoteDatasets\":false,\"namesRefreshMillis\":3600000,\"datasetDefinitionRefreshAfterMillis\":3600000,\"datasetDefinitionExpireAfterMillis\":10800000,\"authTTLMillis\":86400000,\"updateMode\":\"PREFETCH_QUERIED\"}}"

echo "S3 Source created in Dremio"

0 comments on commit 512894b

Please sign in to comment.