forked from DataLinkDC/dinky
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
1 parent
2ca6b0d
commit 40fedad
Showing
20 changed files
with
803 additions
and
114 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Empty file.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,5 @@ | ||
|
|
||
| ARG FLINK_VERSION | ||
|
|
||
| FROM flink:${FLINK_VERSION}-scala_2.12-java8 as flink-base | ||
| RUN rm -f /opt/flink/lib/flink-table-planner-loader*.jar && cp /opt/flink/opt/flink-python*.jar /opt/flink/lib/ && cp /opt/flink/opt/flink-table-planner*.jar /opt/flink/lib/ 2>/dev/null || : |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,11 @@ | ||
| version: "3" | ||
| networks: | ||
| dinky_net: | ||
| external: true | ||
| services: | ||
| doris: | ||
| image: yagagagaga/doris-standalone:2.1.7 | ||
| hostname: doris | ||
| container_name: doris | ||
| networks: | ||
| - dinky_net |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,43 +1,43 @@ | ||
| CORE_CONF_fs_defaultFS=hdfs://namenode:9000 | ||
| CORE_CONF_hadoop_http_staticuser_user=root | ||
| CORE_CONF_hadoop_proxyuser_hue_hosts=* | ||
| CORE_CONF_hadoop_proxyuser_hue_groups=* | ||
| CORE_CONF_io_compression_codecs=org.apache.hadoop.io.compress.SnappyCodec | ||
|
|
||
| HDFS_CONF_dfs_webhdfs_enabled=true | ||
| HDFS_CONF_dfs_permissions_enabled=false | ||
| HDFS_CONF_dfs_namenode_datanode_registration_ip___hostname___check=false | ||
|
|
||
| YARN_CONF_yarn_log___aggregation___enable=true | ||
| YARN_CONF_yarn_log_server_url=http://historyserver:8188/applicationhistory/logs/ | ||
| YARN_CONF_yarn_resourcemanager_recovery_enabled=true | ||
| YARN_CONF_yarn_resourcemanager_store_class=org.apache.hadoop.yarn.server.resourcemanager.recovery.FileSystemRMStateStore | ||
| YARN_CONF_yarn_resourcemanager_scheduler_class=org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler | ||
| YARN_CONF_yarn_scheduler_capacity_root_default_maximum___allocation___mb=8192 | ||
| YARN_CONF_yarn_scheduler_capacity_root_default_maximum___allocation___vcores=4 | ||
| YARN_CONF_yarn_resourcemanager_fs_state___store_uri=/rmstate | ||
| YARN_CONF_yarn_resourcemanager_system___metrics___publisher_enabled=true | ||
| YARN_CONF_yarn_resourcemanager_hostname=resourcemanager | ||
| YARN_CONF_yarn_resourcemanager_address=resourcemanager:8032 | ||
| YARN_CONF_yarn_resourcemanager_scheduler_address=resourcemanager:8030 | ||
| YARN_CONF_yarn_resourcemanager_resource__tracker_address=resourcemanager:8031 | ||
| YARN_CONF_yarn_timeline___service_enabled=true | ||
| YARN_CONF_yarn_timeline___service_generic___application___history_enabled=true | ||
| YARN_CONF_yarn_timeline___service_hostname=historyserver | ||
| YARN_CONF_mapreduce_map_output_compress=true | ||
| YARN_CONF_mapred_map_output_compress_codec=org.apache.hadoop.io.compress.SnappyCodec | ||
| YARN_CONF_yarn_nodemanager_resource_memory___mb=16384 | ||
| YARN_CONF_yarn_nodemanager_resource_cpu___vcores=8 | ||
| YARN_CONF_yarn_nodemanager_disk___health___checker_max___disk___utilization___per___disk___percentage=98.5 | ||
| YARN_CONF_yarn_nodemanager_remote___app___log___dir=/app-logs | ||
| YARN_CONF_yarn_nodemanager_aux___services=mapreduce_shuffle | ||
|
|
||
| MAPRED_CONF_mapreduce_framework_name=yarn | ||
| MAPRED_CONF_mapred_child_java_opts=-Xmx4096m | ||
| MAPRED_CONF_mapreduce_map_memory_mb=4096 | ||
| MAPRED_CONF_mapreduce_reduce_memory_mb=8192 | ||
| MAPRED_CONF_mapreduce_map_java_opts=-Xmx3072m | ||
| MAPRED_CONF_mapreduce_reduce_java_opts=-Xmx6144m | ||
| MAPRED_CONF_yarn_app_mapreduce_am_env=HADOOP_MAPRED_HOME=/data/docker-compose/hadoop-3.2.1/ | ||
| MAPRED_CONF_mapreduce_map_env=HADOOP_MAPRED_HOME=/data/docker-compose/hadoop-3.2.1/ | ||
| MAPRED_CONF_mapreduce_reduce_env=HADOOP_MAPRED_HOME=/data/docker-compose/hadoop-3.2.1/ | ||
| CORE_CONF_fs_defaultFS=hdfs://namenode:9000 | ||
| CORE_CONF_hadoop_http_staticuser_user=root | ||
| CORE_CONF_hadoop_proxyuser_hue_hosts=* | ||
| CORE_CONF_hadoop_proxyuser_hue_groups=* | ||
| CORE_CONF_io_compression_codecs=org.apache.hadoop.io.compress.SnappyCodec | ||
|
|
||
| HDFS_CONF_dfs_webhdfs_enabled=true | ||
| HDFS_CONF_dfs_permissions_enabled=false | ||
| HDFS_CONF_dfs_namenode_datanode_registration_ip___hostname___check=false | ||
|
|
||
| YARN_CONF_yarn_log___aggregation___enable=true | ||
| YARN_CONF_yarn_log_server_url=http://historyserver:8188/applicationhistory/logs/ | ||
| YARN_CONF_yarn_resourcemanager_recovery_enabled=true | ||
| YARN_CONF_yarn_resourcemanager_store_class=org.apache.hadoop.yarn.server.resourcemanager.recovery.FileSystemRMStateStore | ||
| YARN_CONF_yarn_resourcemanager_scheduler_class=org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler | ||
| YARN_CONF_yarn_scheduler_capacity_root_default_maximum___allocation___mb=8192 | ||
| YARN_CONF_yarn_scheduler_capacity_root_default_maximum___allocation___vcores=4 | ||
| YARN_CONF_yarn_resourcemanager_fs_state___store_uri=/rmstate | ||
| YARN_CONF_yarn_resourcemanager_system___metrics___publisher_enabled=true | ||
| YARN_CONF_yarn_resourcemanager_hostname=resourcemanager | ||
| YARN_CONF_yarn_resourcemanager_address=resourcemanager:8032 | ||
| YARN_CONF_yarn_resourcemanager_scheduler_address=resourcemanager:8030 | ||
| YARN_CONF_yarn_resourcemanager_resource__tracker_address=resourcemanager:8031 | ||
| YARN_CONF_yarn_timeline___service_enabled=true | ||
| YARN_CONF_yarn_timeline___service_generic___application___history_enabled=true | ||
| YARN_CONF_yarn_timeline___service_hostname=historyserver | ||
| YARN_CONF_mapreduce_map_output_compress=true | ||
| YARN_CONF_mapred_map_output_compress_codec=org.apache.hadoop.io.compress.SnappyCodec | ||
| YARN_CONF_yarn_nodemanager_resource_memory___mb=163840 | ||
| YARN_CONF_yarn_nodemanager_resource_cpu___vcores=80 | ||
| YARN_CONF_yarn_nodemanager_disk___health___checker_max___disk___utilization___per___disk___percentage=98.5 | ||
| YARN_CONF_yarn_nodemanager_remote___app___log___dir=/app-logs | ||
| YARN_CONF_yarn_nodemanager_aux___services=mapreduce_shuffle | ||
|
|
||
| MAPRED_CONF_mapreduce_framework_name=yarn | ||
| MAPRED_CONF_mapred_child_java_opts=-Xmx4096m | ||
| MAPRED_CONF_mapreduce_map_memory_mb=4096 | ||
| MAPRED_CONF_mapreduce_reduce_memory_mb=8192 | ||
| MAPRED_CONF_mapreduce_map_java_opts=-Xmx3072m | ||
| MAPRED_CONF_mapreduce_reduce_java_opts=-Xmx6144m | ||
| MAPRED_CONF_yarn_app_mapreduce_am_env=HADOOP_MAPRED_HOME=/data/docker-compose/hadoop-3.2.1/ | ||
| MAPRED_CONF_mapreduce_map_env=HADOOP_MAPRED_HOME=/data/docker-compose/hadoop-3.2.1/ | ||
| MAPRED_CONF_mapreduce_reduce_env=HADOOP_MAPRED_HOME=/data/docker-compose/hadoop-3.2.1/ |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,9 +1,63 @@ | ||
| import sys | ||
| from logger import log | ||
|
|
||
| dinky_addr = sys.argv[1] | ||
| flink_version = sys.argv[2] | ||
| dinky_app_jar = 'dinky-app.jar' | ||
|
|
||
| # standalone | ||
| standalone_address = "jobmanager:8282" | ||
|
|
||
| # yarn | ||
| yarn_flink_lib="/flink/lib" | ||
| yarn_flink_conf="/flink/conf" | ||
| yarn_hadoop_conf="/flink/conf" | ||
| yarn_dinky_app_jar="/dinky/jar" | ||
| yarn_flink_lib = "/opt/flink/lib" | ||
| yarn_flink_conf = "/opt/flink/conf" | ||
| yarn_hadoop_conf = "/opt/flink/conf" | ||
| yarn_dinky_app_jar = "/dinky/jar" | ||
|
|
||
| podTemplate=""" | ||
| apiVersion: v1 | ||
| kind: Pod | ||
| metadata: | ||
| name: jobmanager-pod-template | ||
| spec: | ||
| initContainers: | ||
| - name: artifacts-fetcher-dinky | ||
| image: library/busybox:latest | ||
| imagePullPolicy: Never | ||
| # Use wget or other tools to get user jars from remote storage | ||
| command: [ 'wget', 'http://172.28.0.1:9001/dinky-app.jar', '-O', '/flink-usrlib/dinky-app.jar' ] | ||
| volumeMounts: | ||
| - mountPath: /flink-usrlib | ||
| name: flink-usrlib | ||
| - name: artifacts-fetcher-mysql | ||
| image: library/busybox:latest | ||
| imagePullPolicy: Never | ||
| # Use wget or other tools to get user jars from remote storage | ||
| command: [ 'wget', 'http://172.28.0.1:9001/mysql-connector-java-8.0.30.jar', '-O', '/flink-usrlib/mysql-connector-java-8.0.30.jar' ] | ||
| volumeMounts: | ||
| - mountPath: /flink-usrlib | ||
| name: flink-usrlib | ||
| containers: | ||
| # Do not change the main container name | ||
| - name: flink-main-container | ||
| resources: | ||
| requests: | ||
| ephemeral-storage: 2048Mi | ||
| limits: | ||
| ephemeral-storage: 2048Mi | ||
| volumeMounts: | ||
| - mountPath: /opt/flink/usrlib | ||
| name: flink-usrlib | ||
| volumes: | ||
| - name: flink-usrlib | ||
| emptyDir: { } | ||
| """ | ||
|
|
||
| log.info(f""" | ||
| ==================================================== | ||
| all config dinky address: {dinky_addr} | ||
| flink version: {flink_version} | ||
| ==================================================== | ||
| """) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,9 @@ | ||
| set 'execution.checkpointing.interval'='5 s'; | ||
| ADD CUSTOMJAR 'rs:/flink-sql-connector-mysql-cdc.jar'; | ||
| ADD CUSTOMJAR 'rs:/mysql-connector-java.jar'; | ||
| EXECUTE JAR WITH ( | ||
| 'uri'='rs:/flink-doris-connector.jar', | ||
| 'main-class'='org.apache.doris.flink.tools.cdc.CdcTools', | ||
| 'args'='base64@bXlzcWwtc3luYy1kYXRhYmFzZSAgICAgLS1kYXRhYmFzZSBkaW5reSAgICAgLS1teXNxbC1jb25mIGhvc3RuYW1lPW15c3FsICAgICAtLW15c3FsLWNvbmYgcG9ydD0zMzA2ICAgICAtLW15c3FsLWNvbmYgdXNlcm5hbWU9cm9vdCAgICAgLS1teXNxbC1jb25mIHBhc3N3b3JkPWRpbmt5ICAgICAtLW15c3FsLWNvbmYgZGF0YWJhc2UtbmFtZT1kaW5reSAgICAgLS1teXNxbC1jb25mIHNlcnZlci10aW1lLXpvbmU9QXNpYS9TaGFuZ2hhaSAgICAgLS1pbmNsdWRpbmctdGFibGVzICJkaW5reV90YXNrIiAgICAgLS1zaW5rLWNvbmYgZmVub2Rlcz1kb3Jpcy1mZTo4MDMwICAgICAtLXNpbmstY29uZiB1c2VybmFtZT1yb290ICAgICAtLXNpbmstY29uZiBqZGJjLXVybD1qZGJjOm15c3FsOi8vZG9yaXMtZmU6OTAzMCAgICAgLS1zaW5rLWNvbmYgc2luay5sYWJlbC1wcmVmaXg9bGFiZWwtMSAgICAgLS10YWJsZS1jb25mIHJlcGxpY2F0aW9uX251bT0xIA==', | ||
| 'allowNonRestoredState'='false' | ||
| ); |
Oops, something went wrong.