From cf581019aeb1d8f37f65a1494efd854dfbbccb59 Mon Sep 17 00:00:00 2001 From: ataseren Date: Tue, 19 Mar 2024 16:49:19 +0300 Subject: [PATCH] Changes in readthedocs --- .goreleaser.yaml | 2 +- docs/source/conf.py | 2 +- docs/source/contrib.rst | 24 +++- docs/source/docker.rst | 201 ++++++++++++++++++++++++------- docs/source/yaml_format.rst | 228 ++++++++++++++++++------------------ 5 files changed, 293 insertions(+), 164 deletions(-) diff --git a/.goreleaser.yaml b/.goreleaser.yaml index e2174c9..126a9a4 100644 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -32,7 +32,7 @@ changelog: # brews: # - repository: -# owner: yohamta +# owner: erdemozgen # name: homebrew-tap # folder: Formula # homepage: 'https://github.com/erdemozgen/blackdagger' diff --git a/docs/source/conf.py b/docs/source/conf.py index f8b02f3..d17736a 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -9,7 +9,7 @@ project = 'Blackdagger' copyright = '2024, Blackdagger Developers' author = 'Blackdagger Developers' -release = '1.11' +release = '1.03' # -- General configuration --------------------------------------------------- # https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration diff --git a/docs/source/contrib.rst b/docs/source/contrib.rst index 4d70cdb..ccec3f6 100644 --- a/docs/source/contrib.rst +++ b/docs/source/contrib.rst @@ -50,21 +50,33 @@ Running Tests Code Structure --------------- -- ``ui``: Frontend code for the Web UI. - ``cmd``: Contains the main application entry point. - ``docs``: Contains the documentation for the project. - ``examples``: Contains the example workflows. +- ``models``: Contains code related to data models and structures that represent the data used within the application. +- ``restapi``: Contains code related to implementing a RESTful API +- ``schemas``: Contains code that determines YAML format. +- ``scripts``: Contains scripts for download and installation of Blackdagger itself and extra utilities. +- ``service``: TBU +- ``ui``: Frontend code for the Web UI. - ``internal``: Contains the internal code for the project. - - - ``web``: Contains the backend code for the Web UI. - ``agent``: Contains the code for runnning the workflows. - ``config``: Contains the code for loading the configuration. - - ``controller``: Contains the code for managing the workflows. + - ``constants``: - ``dag``: Contains the code for parsing the workflow definition. - - ``database``: Contains the code for interacting with the database. + - ``engine``: Contains the for running basic functions of Blackdagger. + - ``errors``: Contains the code for error handling in commands of DAG. - ``executor``: Contains the code for different types of executors. - - ``runner``: Contains the code for scheduler. + - ``grep``: TBU + - ``logger``: Contains the code for logging functions. + - ``mailer``: Contains the code for mail functions. + - ``pb``: Contains the code for steps in DAG. + - ``persistence``: Contains the code for storage and databases. + - ``proto``: Contains the code for "Protocol Buffers". + - ``reporter``: Contains the code for reporting the status of the scheduler. + - ``scheduler``: Contains the code for scheduler which schedules and runs the processes in a DAG. - ``sock``: Contains the code for interacting with the socket. + - ``utils``: Contains the code for utility functions. Setting up your local environment for front end development ------------------------------------------------------------- diff --git a/docs/source/docker.rst b/docs/source/docker.rst index a3a1009..ced3f77 100644 --- a/docs/source/docker.rst +++ b/docs/source/docker.rst @@ -5,49 +5,164 @@ Create the ``Dockerfile`` and you can build an image. .. code-block:: dockerfile - # syntax=docker/dockerfile:1.4 - FROM --platform=$BUILDPLATFORM alpine:latest - - ARG TARGETARCH - ARG VERSION= - ARG RELEASES_URL="https://github.com/ErdemOzgen/blackdagger/releases" - - ARG USER="blackdagger" - ARG USER_UID=1000 - ARG USER_GID=$USER_UID - - EXPOSE 8080 - - RUN < /etc/sudoers.d/${USER} - chmod 0440 /etc/sudoers.d/${USER} - EOF - - USER blackdagger - WORKDIR /home/blackdagger - RUN <> ~/.bashrc + + # Step 6: Set the working directory + WORKDIR /work_dir + + # Step 7: Copy the file and folders into the container + #COPY . . + COPY ./entrypoint.sh entrypoint.sh + COPY ./startservices.sh startservices.sh + COPY ./update_telegram_config.sh /usr/local/bin/update_telegram_config + COPY ./provider-config.yaml /root/.config/notify/provider-config.yaml + + + # RUN wget https://repo.anaconda.com/archive/Anaconda3-2021.05-Linux-x86_64.sh && \ + # chmod +x Anaconda3-2021.05-Linux-x86_64.sh && \ + # ./Anaconda3-2021.05-Linux-x86_64.sh -b -p /opt/anaconda3 && \ + # rm Anaconda3-2021.05-Linux-x86_64.sh + + # Use a separate stage for runtime to keep the final image smaller + FROM blackarchlinux/blackarch:latest AS runtime + + # Copy the Anaconda installation from the build stage + #COPY --from=build /opt/anaconda3 /opt/anaconda3 + #Copy all binaries from the builder image to the runtime image + COPY --from=build /root/go/bin /root/go/bin + COPY --from=build /usr/local/bin /usr/local/bin + COPY --from=build /usr/local/sbin /usr/local/sbin + COPY --from=build /usr/bin /usr/bin + COPY --from=build /usr/sbin /usr/sbin + COPY --from=build /go/src/app /go/src/app + COPY --from=build /usr /usr + COPY --from=build /lib /lib + COPY --from=build /lib64 /lib64 + COPY --from=build /opt /opt + #COPY --from=build / / + + # Initialize keyring and populate Arch Linux keyring + RUN pacman-key --init && pacman-key --populate archlinux + + # Update the Arch Linux keyring and upgrade the system + #RUN pacman -Sy --noconfirm archlinux-keyring && pacman -Syu --noconfirm + # Set the PATH for Miniconda + #RUN echo 'export PATH=$PATH:/opt/anaconda3/bin' >> ~/.bashrc + + RUN pacman -Sy --noconfirm --overwrite '*' jre11-openjdk + RUN pacman -Sy --noconfirm --overwrite '*' jdk11-openjdk + WORKDIR /work_dir + + # For WebAnalyzer pull this docker and run as API endpoint ==> docker pull erdemozgen/wap_api + # Set the entry point to /bin/bash + RUN echo 'export PATH="/root/go/bin:/sbin:/usr/bin:/root/bin:/usr/sbin:/usr/local/bin:/usr/local/sbin:/opt/bin:/usr/bin/core_perl:$PATH"' >> ~/.bashrc + RUN python -m venv blackdaggerenv + RUN echo 'source blackdaggerenv/bin/activate' >> ~/.bashrc + RUN echo "alias install='pacman -S --noconfirm --overwrite \"*\"'" >> ~/.bashrc + RUN echo "alias update='pacman -Syu --noconfirm --overwrite \"*\"'" >> ~/.bashrc + RUN echo "alias remove='pacman -R --noconfirm'" >> ~/.bashrc + RUN echo "alias search='pacman -Ss'" >> ~/.bashrc + RUN source ~/.bashrc + RUN pacman -Sy --noconfirm --overwrite '*' openssh + # Generate SSH host keys + RUN ssh-keygen -A + + RUN wget https://github.com/yudai/gotty/releases/download/v1.0.1/gotty_linux_amd64.tar.gz -O gotty.tar.gz \ + && tar -xzf gotty.tar.gz \ + && mv gotty /usr/local/bin/ \ + && rm gotty.tar.gz + + + # Generate a self-signed SSL certificate + RUN openssl req -x509 -newkey rsa:4096 -keyout key.pem -out cert.pem -days 365 -nodes -subj '/CN=localhost' + COPY ./provider-config.yaml /root/.config/notify/provider-config.yaml + RUN echo 'export JAVA_HOME=/usr/lib/jvm/java-11-openjdk' >> ~/.bashrc + RUN echo 'export PATH=$JAVA_HOME/bin:$PATH' >> ~/.bashrc + RUN source ~/.bashrc + COPY ./update_telegram_config.sh /usr/local/bin/update_telegram_config + RUN chmod +x /usr/local/bin/update_telegram_config + + # Move the certificate and key to a specific directory (optional) + RUN mkdir -p /etc/gotty && mv cert.pem key.pem /etc/gotty/ + RUN mkdir -p /work_dir/scan_data + RUN source ~/.bashrc + + # Set blackdagger user password + # ARG USER="blackdagger" + # ARG USER_UID=1000 + # ARG USER_GID=$USER_UID + ENV BLACKDAGGER_HOST=0.0.0.0 + ENV BLACKDAGGER_PORT=8080 + + # RUN /bin/bash -c ' \ + # # Update the system and install sudo, handling file conflicts \ + # pacman -Syu --noconfirm --overwrite "*" && \ + # pacman -S --noconfirm --overwrite "*" sudo && \ + # # Clean the package cache to reduce image size \ + # pacman -Scc --noconfirm && \ + # # User and permissions setup, checking if group/user already exists \ + # if ! getent group ${USER_GID}; then \ + # groupadd -g ${USER_GID} ${USER}; \ + # fi; \ + # if ! id -u ${USER} > /dev/null 2>&1; then \ + # useradd -m -s /bin/bash -u ${USER_UID} -g ${USER_GID} ${USER}; \ + # fi; \ + # echo "${USER} ALL=(root) NOPASSWD:ALL" > /etc/sudoers.d/${USER} && \ + # chmod 0440 /etc/sudoers.d/${USER} \ + # ' + + + RUN curl -L https://raw.githubusercontent.com/ErdemOzgen/blackdagger/main/scripts/downloader.sh | bash + + EXPOSE 8080 8090 + + COPY ./entrypoint.sh /entrypoint.sh + COPY ./startservices.sh /startservices.sh + COPY update_telegram_config.sh /usr/local/bin/update_telegram_config + RUN mv /work_dir/blackdagger /usr/local/bin/blackdagger + RUN sh -c 'cp /root/go/bin/* /usr/bin/' + RUN source ~/.bashrc + RUN chmod +x /entrypoint.sh + ENTRYPOINT ["/entrypoint.sh"] For example:: diff --git a/docs/source/yaml_format.rst b/docs/source/yaml_format.rst index bde8cbb..f9699a4 100644 --- a/docs/source/yaml_format.rst +++ b/docs/source/yaml_format.rst @@ -6,6 +6,121 @@ YAML Format .. contents:: :local: +All Available Fields for DAGs +------------------------------- + +This section provides a comprehensive list of available fields that can be used to configure DAGs and their steps in detail. Each field serves a specific purpose, enabling granular control over how the DAG runs. The fields include: + +- ``name``: The name of the DAG, which is optional. The default name is the name of the file. +- ``description``: A brief description of the DAG. +- ``schedule``: The execution schedule of the DAG in Cron expression format. +- ``group``: The group name to organize DAGs, which is optional. +- ``tags``: Free tags that can be used to categorize DAGs, separated by commas. +- ``env``: Environment variables that can be accessed by the DAG and its steps. +- ``logDir``: The directory where the standard output is written. The default value is ``${BLACKDAGGER_HOME}/logs/dags``. +- ``restartWaitSec``: The number of seconds to wait after the DAG process stops before restarting it. +- ``histRetentionDays``: The number of days to retain execution history (not for log files). +- ``delaySec``: The interval time in seconds between steps. +- ``maxActiveRuns``: The maximum number of parallel running steps. +- ``params``: The default parameters that can be referred to by ``$1``, ``$2``, and so on. +- ``preconditions``: The conditions that must be met before a DAG or step can run. +- ``mailOn``: Whether to send an email notification when a DAG or step fails or succeeds. +- ``MaxCleanUpTimeSec``: The maximum time to wait after sending a TERM signal to running steps before killing them. +- ``handlerOn``: The command to execute when a DAG or step succeeds, fails, cancels, or exits. +- ``steps``: A list of steps to execute in the DAG. + +In addition, a global configuration file, ``$BLACKDAGGER_HOME/config.yaml``, can be used to gather common settings, such as ``logDir`` or ``env``. + +Note: If ``BLACKDAGGER_HOME`` environment variable is not set, the default path is ``$HOME/.blackdagger/config.yaml``. + +Example: + +.. code-block:: yaml + + name: DAG name + description: run a DAG + schedule: "0 * * * *" + group: DailyJobs + tags: example + env: + - LOG_DIR: ${HOME}/logs + - PATH: /usr/local/bin:${PATH} + logDir: ${LOG_DIR} + restartWaitSec: 60 + histRetentionDays: 3 + delaySec: 1 + maxActiveRuns: 1 + params: param1 param2 + preconditions: + - condition: "`echo $2`" + expected: "param2" + mailOn: + failure: true + success: true + MaxCleanUpTimeSec: 300 + handlerOn: + success: + command: "echo succeed" + failure: + command: "echo failed" + cancel: + command: "echo canceled" + exit: + command: "echo finished" + + +All Available Fields for Steps +-------------------------------- + +Each step can have its own set of configurations, including: + +- ``name``: The name of the step. +- ``description``: A brief description of the step. +- ``dir``: The working directory for the step. +- ``command``: The command and parameters to execute. +- ``stdout``: The file to which the standard output is written. +- ``output``: The variable to which the result is written. +- ``script``: The script to execute. +- ``signalOnStop``: The signal name (e.g., ``SIGINT``) to be sent when the process is stopped. +- ``mailOn``: Whether to send an email notification when the step fails or succeeds. +- ``continueOn``: Whether to continue to the next step, regardless of whether the step failed or not or the preconditions are met or not. +- ``retryPolicy``: The retry policy for the step. +- ``repeatPolicy``: The repeat policy for the step. +- ``preconditions``: The conditions that must be met before a step can run. +- ``depends``: The step depends on the other step. + +Example: + +.. code-block:: yaml + + steps: + - name: some task + description: some task + dir: ${HOME}/logs + command: bash + stdout: /tmp/outfile + ouptut: RESULT_VARIABLE + script: | + echo "any script" + signalOnStop: "SIGINT" + mailOn: + failure: true + success: true + continueOn: + failure: true + skipped: true + retryPolicy: + limit: 2 + intervalSec: 5 + repeatPolicy: + repeat: true + intervalSec: 60 + preconditions: + - condition: "`echo $1`" + expected: "param1" + depends: + - some task name step + Minimal DAG Definition ----------------------- @@ -229,116 +344,3 @@ You can use the `schedule` field to schedule a DAG with Cron expression. See :ref:`scheduler configuration` for more details. -All Available Fields for DAGs -------------------------------- - -This section provides a comprehensive list of available fields that can be used to configure DAGs and their steps in detail. Each field serves a specific purpose, enabling granular control over how the DAG runs. The fields include: - -- ``name``: The name of the DAG, which is optional. The default name is the name of the file. -- ``description``: A brief description of the DAG. -- ``schedule``: The execution schedule of the DAG in Cron expression format. -- ``group``: The group name to organize DAGs, which is optional. -- ``tags``: Free tags that can be used to categorize DAGs, separated by commas. -- ``env``: Environment variables that can be accessed by the DAG and its steps. -- ``logDir``: The directory where the standard output is written. The default value is ``${BLACKDAGGER_HOME}/logs/dags``. -- ``restartWaitSec``: The number of seconds to wait after the DAG process stops before restarting it. -- ``histRetentionDays``: The number of days to retain execution history (not for log files). -- ``delaySec``: The interval time in seconds between steps. -- ``maxActiveRuns``: The maximum number of parallel running steps. -- ``params``: The default parameters that can be referred to by ``$1``, ``$2``, and so on. -- ``preconditions``: The conditions that must be met before a DAG or step can run. -- ``mailOn``: Whether to send an email notification when a DAG or step fails or succeeds. -- ``MaxCleanUpTimeSec``: The maximum time to wait after sending a TERM signal to running steps before killing them. -- ``handlerOn``: The command to execute when a DAG or step succeeds, fails, cancels, or exits. -- ``steps``: A list of steps to execute in the DAG. - -In addition, a global configuration file, ``$BLACKDAGGER_HOME/config.yaml``, can be used to gather common settings, such as ``logDir`` or ``env``. - -Note: If ``BLACKDAGGER_HOME`` environment variable is not set, the default path is ``$HOME/.blackdagger/config.yaml``. - -Example: - -.. code-block:: yaml - - name: DAG name - description: run a DAG - schedule: "0 * * * *" - group: DailyJobs - tags: example - env: - - LOG_DIR: ${HOME}/logs - - PATH: /usr/local/bin:${PATH} - logDir: ${LOG_DIR} - restartWaitSec: 60 - histRetentionDays: 3 - delaySec: 1 - maxActiveRuns: 1 - params: param1 param2 - preconditions: - - condition: "`echo $2`" - expected: "param2" - mailOn: - failure: true - success: true - MaxCleanUpTimeSec: 300 - handlerOn: - success: - command: "echo succeed" - failure: - command: "echo failed" - cancel: - command: "echo canceled" - exit: - command: "echo finished" - -All Available Fields for Steps --------------------------------- - -Each step can have its own set of configurations, including: - -- ``name``: The name of the step. -- ``description``: A brief description of the step. -- ``dir``: The working directory for the step. -- ``command``: The command and parameters to execute. -- ``stdout``: The file to which the standard output is written. -- ``output``: The variable to which the result is written. -- ``script``: The script to execute. -- ``signalOnStop``: The signal name (e.g., ``SIGINT``) to be sent when the process is stopped. -- ``mailOn``: Whether to send an email notification when the step fails or succeeds. -- ``continueOn``: Whether to continue to the next step, regardless of whether the step failed or not or the preconditions are met or not. -- ``retryPolicy``: The retry policy for the step. -- ``repeatPolicy``: The repeat policy for the step. -- ``preconditions``: The conditions that must be met before a step can run. -- ``depends``: The step depends on the other step. - -Example: - -.. code-block:: yaml - - steps: - - name: some task - description: some task - dir: ${HOME}/logs - command: bash - stdout: /tmp/outfile - ouptut: RESULT_VARIABLE - script: | - echo "any script" - signalOnStop: "SIGINT" - mailOn: - failure: true - success: true - continueOn: - failure: true - skipped: true - retryPolicy: - limit: 2 - intervalSec: 5 - repeatPolicy: - repeat: true - intervalSec: 60 - preconditions: - - condition: "`echo $1`" - expected: "param1" - depends: - - some task name step \ No newline at end of file