docker環(huán)境部署domino的詳細過程
說明
Domino項目是一個開源任務(wù)調(diào)度平臺,結(jié)合Apache Airflow使用,使Airflow的web界面更直觀。
項目代碼:https://github.com/Tauffer-Consulting/domino
后續(xù)更新計劃
去掉對GitHub的依賴,適配主流的國內(nèi)外代碼平臺
增加組件自定義配置,適配最新版airflow
創(chuàng)建文件
.env
# Airflow配置 AIRFLOW_UID=50000 AIRFLOW_GID=0 # Domino配置 DOMINO_COMPOSE_DEV=true # GitHub個人賬戶的令牌在GitHub網(wǎng)站的開發(fā)者配置里創(chuàng)建 DOMINO_DEFAULT_PIECES_REPOSITORY_TOKEN=GitHub個人賬戶的令牌 DOMINO_CREATE_DEFAULT_USER=true # 數(shù)據(jù)庫配置 POSTGRES_USER=domino POSTGRES_PASSWORD=domino_password POSTGRES_DB=domino # Redis配置 REDIS_PASSWORD=redis_password # Domino管理員配置,強制內(nèi)置,不支持自定義。 DOMINO_ADMIN_EMAIL=admin@example.com DOMINO_ADMIN_PASSWORD=admin
compose.yaml
注意,需修改domino-rest中的API URL配置。
# Basic Airflow cluster configuration for CeleryExecutor with Redis and PostgreSQL.
# WARNING: This configuration is for local development. Do not use it in a production deployment.
---
x-airflow-common:
&airflow-common
image: apache/airflow:2.7.2-python3.9
environment:
&airflow-common-env
AIRFLOW__CORE__EXECUTOR: CeleryExecutor
AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: postgresql+psycopg2://airflow:airflow@postgres/airflow
AIRFLOW__CORE__SQL_ALCHEMY_CONN: postgresql+psycopg2://airflow:airflow@postgres/airflow
AIRFLOW__CELERY__RESULT_BACKEND: db+postgresql://airflow:airflow@postgres/airflow
AIRFLOW__CELERY__BROKER_URL: redis://:@redis:6379/0
AIRFLOW__CORE__FERNET_KEY: ''
AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION: 'true'
AIRFLOW__CORE__LOAD_EXAMPLES: 'false'
AIRFLOW__API__AUTH_BACKENDS: 'airflow.api.auth.backend.basic_auth,airflow.api.auth.backend.session'
AIRFLOW__SCHEDULER__ENABLE_HEALTH_CHECK: 'true'
AIRFLOW__SCHEDULER__DAG_DIR_LIST_INTERVAL: 10
_PIP_ADDITIONAL_REQUIREMENTS: ${_PIP_ADDITIONAL_REQUIREMENTS:-}
volumes:
- ${AIRFLOW_PROJ_DIR:-./airflow}/dags:/opt/airflow/dags
- ${AIRFLOW_PROJ_DIR:-./airflow}/logs:/opt/airflow/logs
- ${AIRFLOW_PROJ_DIR:-./airflow}/plugins:/opt/airflow/plugins
user: "${AIRFLOW_UID:-50000}:0"
depends_on:
&airflow-common-depends-on
redis:
condition: service_healthy
postgres:
condition: service_healthy
services:
postgres:
image: postgres:13
container_name: airflow-postgres
environment:
POSTGRES_USER: airflow
POSTGRES_PASSWORD: airflow
POSTGRES_DB: airflow
volumes:
- airflow_postgres_data:/var/lib/postgresql/data
healthcheck:
test: [ "CMD", "pg_isready", "-U", "airflow" ]
interval: 10s
retries: 5
start_period: 5s
restart: always
networks:
- domino-net
redis:
image: redis:latest
container_name: airflow-redis
expose:
- 6379
healthcheck:
test: [ "CMD", "redis-cli", "ping" ]
interval: 10s
timeout: 30s
retries: 50
start_period: 30s
restart: always
networks:
- domino-net
airflow-webserver:
<<: *airflow-common
container_name: airflow-webserver
command: webserver
ports:
- "8080:8080"
healthcheck:
test:
[
"CMD",
"curl",
"--fail",
"http://localhost:8080/health"
]
interval: 30s
timeout: 10s
retries: 5
start_period: 30s
restart: always
depends_on:
<<: *airflow-common-depends-on
airflow-init:
condition: service_completed_successfully
networks:
- domino-net
airflow-triggerer:
<<: *airflow-common
container_name: airflow-triggerer
command: triggerer
healthcheck:
test:
[
"CMD-SHELL",
'airflow jobs check --job-type TriggererJob --hostname "$${HOSTNAME}"'
]
interval: 30s
timeout: 10s
retries: 5
start_period: 30s
restart: always
depends_on:
<<: *airflow-common-depends-on
airflow-init:
condition: service_completed_successfully
networks:
- domino-net
airflow-init:
<<: *airflow-common
container_name: airflow-init
entrypoint: /bin/bash
command:
- -c
- |
function ver() {
printf "%04d%04d%04d%04d" $${1//./ }
}
airflow_version=$$(AIRFLOW__LOGGING__LOGGING_LEVEL=INFO && gosu airflow airflow version)
airflow_version_comparable=$$(ver $${airflow_version})
min_airflow_version=2.2.0
min_airflow_version_comparable=$$(ver $${min_airflow_version})
if (( airflow_version_comparable < min_airflow_version_comparable )); then
echo
echo -e "\033[1;31mERROR!!!: Too old Airflow version $${airflow_version}!\e[0m"
echo "The minimum Airflow version supported: $${min_airflow_version}. Only use this or higher!"
echo
exit 1
fi
if [[ -z "${AIRFLOW_UID}" ]]; then
echo
echo -e "\033[1;33mWARNING!!!: AIRFLOW_UID not set!\e[0m"
echo "If you are on Linux, you SHOULD follow the instructions below to set "
echo "AIRFLOW_UID environment variable, otherwise files will be owned by root."
echo "For other operating systems you can get rid of the warning with manually created .env file:"
echo " See: https://airflow.apache.org/docs/apache-airflow/stable/howto/docker-compose/index.html#setting-the-right-airflow-user"
echo
fi
one_meg=1048576
mem_available=$$(($$(getconf _PHYS_PAGES) * $$(getconf PAGE_SIZE) / one_meg))
cpus_available=$$(grep -cE 'cpu[0-9]+' /proc/stat)
disk_available=$$(df / | tail -1 | awk '{print $$4}')
warning_resources="false"
if (( mem_available < 4000 )) ; then
echo
echo -e "\033[1;33mWARNING!!!: Not enough memory available for Docker.\e[0m"
echo "At least 4GB of memory required. You have $$(numfmt --to iec $$((mem_available * one_meg)))"
echo
warning_resources="true"
fi
if (( cpus_available < 2 )); then
echo
echo -e "\033[1;33mWARNING!!!: Not enough CPUS available for Docker.\e[0m"
echo "At least 2 CPUs recommended. You have $${cpus_available}"
echo
warning_resources="true"
fi
if (( disk_available < one_meg * 10 )); then
echo
echo -e "\033[1;33mWARNING!!!: Not enough Disk space available for Docker.\e[0m"
echo "At least 10 GBs recommended. You have $$(numfmt --to iec $$((disk_available * 1024 )))"
echo
warning_resources="true"
fi
if [[ $${warning_resources} == "true" ]]; then
echo
echo -e "\033[1;33mWARNING!!!: You have not enough resources to run Airflow (see above)!\e[0m"
echo "Please follow the instructions to increase amount of resources available:"
echo " https://airflow.apache.org/docs/apache-airflow/stable/howto/docker-compose/index.html#before-you-begin"
echo
fi
mkdir -p /sources/logs /sources/dags /sources/plugins
chown -R "${AIRFLOW_UID}:0" /sources/{logs,dags,plugins}
exec /entrypoint airflow version
environment:
<<: *airflow-common-env
_AIRFLOW_DB_UPGRADE: 'true'
_AIRFLOW_WWW_USER_CREATE: 'true'
_AIRFLOW_WWW_USER_USERNAME: ${_AIRFLOW_WWW_USER_USERNAME:-airflow}
_AIRFLOW_WWW_USER_PASSWORD: ${_AIRFLOW_WWW_USER_PASSWORD:-airflow}
_PIP_ADDITIONAL_REQUIREMENTS: ''
user: "0:0"
volumes:
- ${AIRFLOW_PROJ_DIR:-./airflow}:/sources
networks:
- domino-net
airflow-cli:
<<: *airflow-common
container_name: airflow-cli
profiles:
- debug
environment:
<<: *airflow-common-env
CONNECTION_CHECK_MAX_COUNT: "0"
command:
- bash
- -c
- airflow
networks:
- domino-net
# 不用可注釋掉
flower:
<<: *airflow-common
container_name: airflow-flower
command: celery flower
#profiles:
# - flower
ports:
- "5555:5555"
healthcheck:
test: [ "CMD", "curl", "--fail", "http://localhost:5555/" ]
interval: 30s
timeout: 10s
retries: 5
start_period: 30s
restart: always
depends_on:
<<: *airflow-common-depends-on
airflow-init:
condition: service_completed_successfully
networks:
- domino-net
# Modified Airflow Scheduler with Domino
airflow-scheduler:
<<: *airflow-common
image: ghcr.io/tauffer-consulting/domino-airflow-base:latest
container_name: airflow-scheduler
command: scheduler
healthcheck:
test:
[
"CMD",
"curl",
"--fail",
"http://localhost:8974/health"
]
interval: 30s
timeout: 10s
retries: 5
start_period: 30s
restart: always
environment:
<<: *airflow-common-env
DOMINO_DEPLOY_MODE: local-compose
volumes:
- ${AIRFLOW_PROJ_DIR:-./airflow}/dags:/opt/airflow/dags
- ${AIRFLOW_PROJ_DIR:-./airflow}/logs:/opt/airflow/logs
- ${AIRFLOW_PROJ_DIR:-./airflow}/plugins:/opt/airflow/plugins
depends_on:
<<: *airflow-common-depends-on
airflow-init:
condition: service_completed_successfully
networks:
- domino-net
# Modified Airflow Worker with Domino
airflow-worker:
<<: *airflow-common
image: ghcr.io/tauffer-consulting/domino-airflow-base:latest
container_name: airflow-worker
command: celery worker
healthcheck:
test:
- "CMD-SHELL"
- 'celery --app airflow.executors.celery_executor.app inspect ping -d "celery@$${HOSTNAME}"'
interval: 30s
timeout: 10s
retries: 5
start_period: 30s
environment:
<<: *airflow-common-env
DUMB_INIT_SETSID: "0"
DOMINO_DEPLOY_MODE: local-compose
LOCAL_DOMINO_SHARED_DATA_PATH: ${PWD}/domino_data
restart: always
volumes:
- ${AIRFLOW_PROJ_DIR:-./airflow}/dags:/opt/airflow/dags
- ${AIRFLOW_PROJ_DIR:-./airflow}/logs:/opt/airflow/logs
- ${AIRFLOW_PROJ_DIR:-./airflow}/plugins:/opt/airflow/plugins
- ${PWD}/domino_data:/home/shared_storage
depends_on:
<<: *airflow-common-depends-on
airflow-init:
condition: service_completed_successfully
networks:
- domino-net
# Domino REST Api
domino_rest:
image: zhuyifeiruichuang/domino-rest:latest
container_name: domino-rest
command: bash -c "alembic upgrade heads && python -m utils.populate_first_user && uvicorn main:app --reload --workers 1 --host 0.0.0.0 --port 8000"
ports:
- 8000:8000
environment:
- DOMINO_DB_USER=postgres
- DOMINO_DB_PASSWORD=postgres
- DOMINO_DB_HOST=domino_postgres
- DOMINO_DB_PORT=5432
- DOMINO_DB_NAME=postgres
- DOMINO_DEFAULT_PIECES_REPOSITORY_TOKEN=${DOMINO_DEFAULT_PIECES_REPOSITORY_TOKEN}
- DOMINO_DEPLOY_MODE=local-compose
- AIRFLOW_ADMIN_USERNAME=airflow
- AIRFLOW_ADMIN_PASSWORD=airflow
- CREATE_DEFAULT_USER=${DOMINO_CREATE_DEFAULT_USER}
networks:
- domino-net
volumes:
- ${AIRFLOW_PROJ_DIR:-./airflow}/dags:/opt/airflow/dags
depends_on:
domino_postgres:
condition: service_healthy
# Domino Postgres
domino_postgres:
image: postgres:13
container_name: domino-postgres
environment:
- POSTGRES_DB=postgres
- POSTGRES_USER=postgres
- POSTGRES_PASSWORD=postgres
volumes:
- domino_postgres_data:/var/lib/postgresql/data
networks:
- domino-net
healthcheck:
test:
[
"CMD-SHELL",
"pg_isready -d $${POSTGRES_DB} -U $${POSTGRES_USER}"
]
interval: 10s
timeout: 60s
retries: 5
start_period: 2s
ports:
- "5433:5432"
restart: always
# Domino Frontend
domino_frontend:
image: zhuyifeiruichuang/domino-frontend:compose
container_name: domino-frontend
ports:
- "3000:80"
depends_on:
domino_rest:
condition: service_started
environment:
- DOMINO_DEPLOY_MODE=local-compose
# 此處只能填寫最終訪問的IP和端口,無法識別容器名字。EIP場景填寫EIP,NAT場景填寫轉(zhuǎn)發(fā)后的IP和端口。
- API_URL=http://IP:8000
networks:
- domino-net
docker-proxy:
image: alpine/socat
container_name: domino-docker-proxy
command: "TCP4-LISTEN:2375,fork,reuseaddr UNIX-CONNECT:/var/run/docker.sock"
ports:
- "2376:2375"
volumes:
- /var/run/docker.sock:/var/run/docker.sock
networks:
- domino-net
volumes:
airflow_postgres_data:
name: airflow_postgres_data
domino_postgres_data:
name: domino_postgres_data
networks:
domino-net:
name: domino-network
driver: bridge部署
docker compose up -d
訪問
訪問dominoIP:3000 賬戶admin@example.com,密碼admin
訪問配套的airflowIP:8080 賬戶密碼是airflow
到此這篇關(guān)于docker環(huán)境部署domino的詳細過程的文章就介紹到這了,更多相關(guān)docker部署domino內(nèi)容請搜索腳本之家以前的文章或繼續(xù)瀏覽下面的相關(guān)文章希望大家以后多多支持腳本之家!
相關(guān)文章
Windows的docker刪除容器后WSL2磁盤空間不釋放的問題的解決方法
很多同學(xué)拉取鏡像使用一段時間后發(fā)現(xiàn) C 盤快滿了,把之前用過的鏡像和容器刪除,發(fā)現(xiàn) WSL 掛載目錄的虛擬磁盤大小沒有變化,非常的奇怪,所以本文介紹了Windows的docker刪除容器后WSL2磁盤空間不釋放的問題的解決方法,需要的朋友可以參考下2024-12-12
Docker基礎(chǔ)學(xué)習(xí)之?dāng)?shù)據(jù)管理
我們在使用Docker 的時候,會產(chǎn)生很多數(shù)據(jù),比如web服務(wù)器啊,數(shù)據(jù)庫之類的,有時我們還需要備份或復(fù)制這些數(shù)據(jù),這就需要涉及到Docker的數(shù)據(jù)管理了。這篇文章就給大家詳細的介紹Docker的數(shù)據(jù)管理,感興趣的朋友們可以參考借鑒,下面來一起看看吧。2016-10-10
Jenkins打包微服務(wù)構(gòu)建Docker鏡像運行的實現(xiàn)
本文主要介紹了Jenkins打包微服務(wù)構(gòu)建Docker鏡像運行的實現(xiàn),文中通過示例代碼介紹的非常詳細,具有一定的參考價值,感興趣的小伙伴們可以參考一下2021-09-09

