From 5ce722c77ec04cd39e42ca17908eec8b74078a52 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Mon, 22 Aug 2022 19:43:05 +0000 Subject: [PATCH 1/2] Make framework complete the build process --- ares/docker/logs_register_teams_services.txt | 0 database/api/brand_new_scoring.py | 4 +- database/provisioning/create_services.py | 10 +- database/provisioning/create_teams.py | 10 +- database/requirements.txt | 2 +- logs_build_infra.txt | 244 ++++++++++++++++++ logs_deploy_infra.txt | 11 + logs_register_teams_services.txt | 30 +++ .../ansible-provisioning.yml | 6 +- router/requirements2.txt | 2 +- router/requirements3.txt | 3 +- router/start.sh | 4 +- scoreboard/frontend/package.json | 2 +- .../scoring_ictf/scoring_interface.py | 5 +- teaminterface/requirements.txt | 4 +- 15 files changed, 312 insertions(+), 25 deletions(-) create mode 100644 ares/docker/logs_register_teams_services.txt create mode 100644 logs_build_infra.txt create mode 100644 logs_deploy_infra.txt create mode 100644 logs_register_teams_services.txt diff --git a/ares/docker/logs_register_teams_services.txt b/ares/docker/logs_register_teams_services.txt new file mode 100644 index 0000000..e69de29 diff --git a/database/api/brand_new_scoring.py b/database/api/brand_new_scoring.py index d16ed78..4c7a50d 100644 --- a/database/api/brand_new_scoring.py +++ b/database/api/brand_new_scoring.py @@ -58,10 +58,10 @@ def scores_get(tick_id=None): if tick_id is None: tick_id, _, _, _ = get_current_tick(cursor=mysql.cursor()) - scores_new = scoring.get_scores_for_tick(tick_id - 1) + scores_new = scoring.get_scores_for_tick((tick_id - 1) if tick_id > 0 else 0) if scores_new != scores_old: print("OLD: {}" + str(scores_old)) print("NEW: {}" + str(scores_new)) - return json.dumps({"scores": scores_new}) \ No newline at end of file + return json.dumps({"scores": scores_new}) diff --git a/database/provisioning/create_services.py b/database/provisioning/create_services.py index ce37394..bd1d88a 100644 --- a/database/provisioning/create_services.py +++ b/database/provisioning/create_services.py @@ -20,7 +20,11 @@ def register_service(db_api_url_base, db_secret, service_name, service_info): result = requests.post(db_api_url_base + "/upload/new", data=data, params={'secret': db_secret}) - response = result.json() + try: + response = result.json() + except Exception as ex: + raise Exception("Invalid JSON returned from /upload/new endpoint: {}".format(result.content)) from ex + #response = result.json() if response['result'] == 'success': # print("successfully uploaded bundle") @@ -47,7 +51,7 @@ def register_service(db_api_url_base, db_secret, service_name, service_info): script_type = os.path.basename(script) if script_type in {'setflag', 'getflag', 'benign', 'exploit'}: - print (f"Uploading script {filename} [{script_type}]") + print (f"Uploading script {script} [{script_type}]") data = {"upload_id": upload_id, "filename": script, "type": script_type, "state": service_info['state'], "service_id": service_id} result = requests.post(db_api_url_base + "/script/new", data=data, params={'secret': db_secret}) @@ -71,7 +75,7 @@ def create_service(db_api_url_base, db_secret, service_path, service_state): service_yaml = os.path.abspath(os.path.join(service_path, './info.yaml')) if not os.path.isfile(service_yaml): raise Exception("Could not find service file for %s, tried %s. Skipping." % (service_path, service_yaml)) - service_info = yaml.load(open(service_yaml, 'r')) + service_info = yaml.load(open(service_yaml, 'r'), Loader=yaml.Loader) # Check that the service path matches the name in info.yaml diff --git a/database/provisioning/create_teams.py b/database/provisioning/create_teams.py index 8808ebc..80b36a4 100644 --- a/database/provisioning/create_teams.py +++ b/database/provisioning/create_teams.py @@ -6,6 +6,7 @@ import string import yaml import json +import time rnd = random.SystemRandom() @@ -91,6 +92,8 @@ def add_teams_info(db_api_base_url, db_secret, game_config): if __name__== "__main__": + print("Waiting for database container to finish startup.") + time.sleep(10) game_config = json.load(open(sys.argv[2], 'r')) db_api = sys.argv[1] # passed from terraform script database_api_secret_path = SECRETS_FOLDER+"database-api/secret" @@ -102,10 +105,3 @@ def add_teams_info(db_api_base_url, db_secret, game_config): add_teams_info('http://' + db_api, db_secret, game_config) else: raise Exception("Missing database secrets!") - - - - - - - diff --git a/database/requirements.txt b/database/requirements.txt index b4d06eb..791eea3 100755 --- a/database/requirements.txt +++ b/database/requirements.txt @@ -7,7 +7,7 @@ ndg-httpsclient pyasn1 setuptools-rust wheel -Flask==0.12.2 +Flask==2.2.2 uWSGI==2.0.15 PyYAML==3.12 boto3==1.4.8 diff --git a/logs_build_infra.txt b/logs_build_infra.txt new file mode 100644 index 0000000..ec74157 --- /dev/null +++ b/logs_build_infra.txt @@ -0,0 +1,244 @@ +Step 1/7 : FROM ubuntu:20.04 + ---> 3bc6e9f30f51 +Step 2/7 : RUN DEBIAN_FRONTEND=noninteractive apt-get -y --no-install-recommends -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" upgrade && apt-get update -y && apt-get -y install software-properties-common && apt-add-repository ppa:ansible/ansible && apt-get -y update && apt-get -y install ansible net-tools iputils-ping + ---> Using cache + ---> 3af9ac30c45a +Step 3/7 : RUN apt-get -y install --only-upgrade python3 python3.8 + ---> Using cache + ---> 0071dcd900b1 +Step 4/7 : COPY ./secrets /opt/ictf/secrets + ---> Using cache + ---> 6a875e80505f +Step 5/7 : COPY ./ictf-base/provisioning/ansible-provisioning.yml /root + ---> Using cache + ---> 691c9d613604 +Step 6/7 : COPY ./ictf-base/provisioning/requirements-ansible-roles.yml /tmp + ---> Using cache + ---> 75c2f2eff922 +Step 7/7 : RUN ansible-playbook /root/ansible-provisioning.yml + ---> Using cache + ---> 0cabdeab89dc + +Successfully built 0cabdeab89dc +Successfully tagged ictf_base:latest +Step 1/6 : FROM rabbitmq:management + ---> 321a17c29307 +Step 2/6 : ENV RABBITMQ_PID_FILE /var/lib/rabbitmq/mnesia/rabbitmq + ---> Using cache + ---> 2655892c5868 +Step 3/6 : COPY ./dispatcher /opt/ictf/dispatcher + ---> Using cache + ---> beb9a1b2cee0 +Step 4/6 : WORKDIR /opt/ictf/dispatcher + ---> Using cache + ---> 21b22e96aee7 +Step 5/6 : RUN chmod +x ./start.sh + ---> Using cache + ---> 99a175e979fc +Step 6/6 : ENTRYPOINT ./start.sh + ---> Using cache + ---> 26364e2a1a60 + +Successfully built 26364e2a1a60 +Step 1/9 : FROM ictf_base + ---> 0cabdeab89dc +Step 2/9 : RUN DEBIAN_FRONTEND=noninteractive apt-get install -y python3-pip python3-virtualenv python3 python3-dev git libssl-dev libffi-dev build-essential libc6-dev-i386 + ---> Using cache + ---> 82239dba4157 +Step 3/9 : COPY ./scriptbot /opt/ictf/scriptbot + ---> Using cache + ---> 9d9db15acf4a +Step 4/9 : WORKDIR /opt/ictf/scriptbot + ---> Using cache + ---> c005b43f5cd8 +Step 5/9 : COPY ./common/hephaestus_provisioning/teamhosts ./teamhosts + ---> Using cache + ---> cc5fcb075bf2 +Step 6/9 : RUN pip3 install -r requirements.txt + ---> Using cache + ---> 34016535702f +Step 7/9 : RUN ansible-playbook provisioning/hephaestus_provisioning/ansible-provisioning.yml + ---> Using cache + ---> cacc08817262 +Step 8/9 : RUN chmod +x ./start.sh && chmod +x ./start_idle_mode.sh + ---> Using cache + ---> 76f21420d0c9 +Step 9/9 : ENTRYPOINT ./start.sh + ---> Using cache + ---> d998ad48c56d + +Successfully tagged ictf_dispatcher:latest +Step 1/8 : FROM ictf_base + ---> 0cabdeab89dc +Step 2/8 : RUN DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends python3-pip python3-wheel + ---> Using cache + ---> d096f32f65c1 +Step 3/8 : COPY ./gamebot /opt/ictf/gamebot + ---> Using cache + ---> c592e46a9638 +Step 4/8 : WORKDIR /opt/ictf/gamebot + ---> Using cache + ---> 5b1dd5b61645 +Step 5/8 : RUN pip install -r requirements.txt + ---> Using cache + ---> ffdc3d4b806a +Step 6/8 : RUN chmod +x ./start.sh + ---> Using cache + ---> fbf83d307a12 +Step 7/8 : RUN ansible-playbook ./provisioning/hephaestus_provisioning/ansible-provisioning.yml --extra-vars ICTF_API_ADDRESS="database.ictf" + ---> Using cache + ---> 2c5e97fbd9c6 +Step 8/8 : ENTRYPOINT ./start.sh + ---> Using cache + ---> 6c681b1a486a + +Step 1/13 : FROM ictf_base + ---> 0cabdeab89dc +Step 2/13 : ENV DEBIAN_FRONTEND=noninteractive + ---> Using cache + ---> 2f9d82f3ffd9 +Step 3/13 : RUN apt-get update && apt-get install -y nginx git redis-server build-essential + ---> Using cache + ---> 91355d09b17d +Step 4/13 : RUN apt-get update && apt-get install -y sudo uwsgi daemon + ---> Using cache + ---> 9b5578c6dc5e +Step 5/13 : RUN curl -sL https://deb.nodesource.com/setup_14.x | bash - + ---> Using cache + ---> a124d9ec37aa +Step 6/13 : RUN apt-get update && apt-get install -y nodejs + ---> Using cache + ---> 0b54ac056290 +Step 7/13 : RUN apt-get update && apt-get install -y python3-pip python3-virtualenv python3-dev uwsgi-plugin-python3 python3-setuptools python3-wheel + ---> Using cache + ---> 69f6466cf7df +Step 8/13 : COPY ./scoreboard /opt/ictf/scoreboard + ---> Using cache + ---> 478f66ea1162 +Step 9/13 : WORKDIR /opt/ictf/scoreboard + ---> Using cache + ---> c0ba46f898b6 +Step 10/13 : RUN pip install -r requirements.txt + ---> Using cache + ---> 1e5140cf78b1 +Step 11/13 : RUN ansible-playbook provisioning/hephaestus_provisioning/ansible-provisioning.yml --become --extra-vars ICTF_USER="root" --extra-vars ICTF_FRAMEWORK_DIR_HOST="/opt/ictf" --extra-vars ICTF_API_ADDRESS="database.ictf" + ---> Using cache + ---> 9ed67e11f8c0 +Step 12/13 : RUN chmod +x ./start.sh + ---> Using cache + ---> fa2097412a8f +Step 13/13 : ENTRYPOINT ./start.sh + ---> Using cache + ---> a65b12846144 + +Successfully built d998ad48c56d +Successfully tagged ictf_scriptbot:latest +Successfully built 6c681b1a486a +Successfully built a65b12846144 +Successfully tagged ictf_gamebot:latest +Successfully tagged ictf_scoreboard:latest +Step 1/8 : FROM ictf_base + ---> 0cabdeab89dc +Step 2/8 : RUN DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends iptraf python3 python3-pip python3-dev build-essential awscli iptables-persistent iftop libcurl4-openssl-dev libssl-dev python3-apt zip unzip openvpn python3-setuptools python3-wheel + ---> Using cache + ---> b196a60c4431 +Step 3/8 : COPY ./router /opt/ictf/router + ---> Using cache + ---> 9b99adbbfc25 +Step 4/8 : WORKDIR /opt/ictf/router + ---> Using cache + ---> 809563d1b92a +Step 5/8 : RUN pip install -r requirements3.txt + ---> Using cache + ---> d898a1a95858 +Step 6/8 : RUN ansible-playbook provisioning/hephaestus_provisioning/ansible-provisioning.yml --become + ---> Using cache + ---> 817aeda275ce +Step 7/8 : RUN chmod +x ./start.sh + ---> Using cache + ---> 06d316ec2e21 +Step 8/8 : ENTRYPOINT ./start.sh + ---> Using cache + ---> 6726981fb9f8 + +Successfully built 6726981fb9f8 +Successfully tagged ictf_router:latest +Step 1/8 : FROM ictf_base + ---> 0cabdeab89dc +Step 2/8 : RUN DEBIAN_FRONTEND=noninteractive apt-get install -y nginx uwsgi uwsgi-plugin-python3 mysql-server python3-pip python3-virtualenv python3-mysqldb python3-dev python3-setuptools python3-wheel cron daemon + ---> Using cache + ---> e25385e23b2d +Step 3/8 : RUN pip install --upgrade pip + ---> Using cache + ---> fca8b017f7b8 +Step 4/8 : COPY ./database /opt/ictf/database + ---> Using cache + ---> a4b800b1dd80 +Step 5/8 : COPY ./scoring_ictf /opt/ictf/scoring_ictf + ---> Using cache + ---> 00073f2844f6 +Step 6/8 : WORKDIR /opt/ictf/database + ---> Using cache + ---> 85ef25e9720d +Step 7/8 : RUN chmod +x ./start.sh && ansible-playbook ./provisioning/hephaestus_provisioning/ansible-provisioning.yml + ---> Using cache + ---> 775cd2e600bf +Step 8/8 : ENTRYPOINT ./start.sh + ---> Using cache + ---> 99c4d513a85b + +Successfully built 99c4d513a85b +Successfully tagged ictf_database:latest +Step 1/8 : FROM ubuntu:18.04 + ---> 8d5df41c547b +Step 2/8 : RUN DEBIAN_FRONTEND=noninteractive apt-get -y --no-install-recommends -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" upgrade && apt-get update -y && apt-get -y install software-properties-common && apt-add-repository ppa:ansible/ansible && apt-get -y update && apt -y install ansible git wget curl vim iputils-ping python-pip python-dev python3-dev build-essential htop + ---> Using cache + ---> 7b4045115698 +Step 3/8 : COPY ./secrets /opt/ictf/secrets + ---> Using cache + ---> 87cc403ef32f +Step 4/8 : COPY ./logger /opt/ictf/logger + ---> Using cache + ---> a1753612b039 +Step 5/8 : WORKDIR /opt/ictf/logger + ---> Using cache + ---> 186899b3a7d7 +Step 6/8 : RUN ansible-playbook provisioning/hephaestus_provisioning/ansible-provisioning.yml --become --extra-vars ICTF_USER="root" --extra-vars ICTF_FRAMEWORK_DIR_HOST="/opt/ictf" + ---> Using cache + ---> 8dc50ee7cb67 +Step 7/8 : RUN chmod +x ./start.sh + ---> Using cache + ---> 960bc50a42d7 +Step 8/8 : ENTRYPOINT ./start.sh + ---> Using cache + ---> 554f7987b11b + +Step 1/8 : FROM ictf_base + ---> 0cabdeab89dc +Step 2/8 : RUN DEBIAN_FRONTEND=noninteractive apt-get install -y nginx python3-pip python3-virtualenv python3-dev libjpeg8 libjpeg-dev zlib1g zlib1g-dev libpng-dev libmemcached-dev libmemcached-tools libgeoip1 libgeoip-dev geoip-bin geoip-database redis-server + ---> Using cache + ---> 6fcde0df0a58 +Step 3/8 : COPY ./teaminterface /opt/ictf/teaminterface + ---> Using cache + ---> f3621945ca6c +Step 4/8 : WORKDIR /opt/ictf/teaminterface + ---> Using cache + ---> cc01211f96da +Step 5/8 : RUN pip install -r requirements.txt + ---> Using cache + ---> fe28c9ff298c +Step 6/8 : RUN ansible-playbook provisioning/hephaestus_provisioning/ansible-provisioning.yml --become --extra-vars ICTF_DB_API_ADDRESS="database.ictf" + ---> Using cache + ---> c6cd00b82763 +Step 7/8 : RUN chmod +x ./start.sh + ---> Using cache + ---> 92e9744ef0a2 +Step 8/8 : ENTRYPOINT ./start.sh + ---> Using cache + ---> e2cda82891b0 + +Successfully built 554f7987b11b +Successfully tagged ictf_logger:latest +Successfully built e2cda82891b0 +Successfully tagged ictf_teaminterface:latest +/home/ubuntu/ictf-framework diff --git a/logs_deploy_infra.txt b/logs_deploy_infra.txt new file mode 100644 index 0000000..6d7dba2 --- /dev/null +++ b/logs_deploy_infra.txt @@ -0,0 +1,11 @@ + +Configuration for docker-compose successfully generated in ./docker-compose-local.generated.yml + +Spawn the infrastructure locally with the following command: + - docker-compose -f docker-compose-local.generated.yml up + +Destroy the infrastructure locally with the following command: + - docker-compose -f docker-compose-local.generated.yml down -v --remove-orphans + +To start the game: http://localhost:5000/game/insert?secret=jxCsP8b6XE5HYY2Do_bTGoknO + diff --git a/logs_register_teams_services.txt b/logs_register_teams_services.txt new file mode 100644 index 0000000..14f5060 --- /dev/null +++ b/logs_register_teams_services.txt @@ -0,0 +1,30 @@ +Waiting for database container to finish startup. +http://127.0.0.1:5000/teams/info +{'teams': {}} +b'{"result": "success", "team_id": 1, "fail_reason": null}' +Team 1 successfully created and validated +b'{"result": "success", "team_id": 2, "fail_reason": null}' +Team 2 successfully created and validated +/home/ubuntu/ictf-test-ctf-1-master/atm_machine +Uploading script setflag [setflag] +Unknown script type Dockerfile skipping Dockerfile +Unknown script type ._accounts.txt skipping ._accounts.txt +Uploading script benign [benign] +Uploading script exploit [exploit] +Unknown script type test_everything.py skipping test_everything.py +Unknown script type createDB.py skipping createDB.py +Uploading script getflag [getflag] +Unknown script type accounts.txt skipping accounts.txt +/home/ubuntu/ictf-test-ctf-1-master/sharing +Uploading script setflag [setflag] +Unknown script type Dockerfile skipping Dockerfile +Uploading script benign [benign] +Uploading script exploit [exploit] +Uploading script getflag [getflag] +/home/ubuntu/ictf-test-ctf-1-master/tweety_bird +Uploading script setflag [setflag] +Unknown script type Dockerfile skipping Dockerfile +Uploading script benign [benign] +Uploading script exploit [exploit] +Uploading script getflag [getflag] +/home/ubuntu/ictf-framework diff --git a/router/provisioning/hephaestus_provisioning/ansible-provisioning.yml b/router/provisioning/hephaestus_provisioning/ansible-provisioning.yml index a59193f..60ce5b1 100644 --- a/router/provisioning/hephaestus_provisioning/ansible-provisioning.yml +++ b/router/provisioning/hephaestus_provisioning/ansible-provisioning.yml @@ -11,12 +11,12 @@ apt_key: id: 30EBF4E73CCE63EEE124DD278E6DA8B4E158C569 url: https://swupdate.openvpn.net/repos/repo-public.gpg - when: ansible_distribution == 'Ubuntu' and ansible_distribution_release == 'xenial' + when: ansible_distribution == 'Ubuntu' and ansible_distribution_release == 'focal' - name: add openvpn repo apt_repository: - repo: deb http://build.openvpn.net/debian/openvpn/release/2.4 xenial main - when: ansible_distribution == 'Ubuntu' and ansible_distribution_release == 'xenial' + repo: deb http://build.openvpn.net/debian/openvpn/release/2.4 focal main + when: ansible_distribution == 'Ubuntu' and ansible_distribution_release == 'focal' - name: install openvpn apt: diff --git a/router/requirements2.txt b/router/requirements2.txt index 485a9db..545f2b9 100755 --- a/router/requirements2.txt +++ b/router/requirements2.txt @@ -1,5 +1,5 @@ setuptools-rust -cryptography==3.2 +cryptography==3.3.2 wheel requests awscli diff --git a/router/requirements3.txt b/router/requirements3.txt index 69a8304..7e36355 100755 --- a/router/requirements3.txt +++ b/router/requirements3.txt @@ -1,5 +1,5 @@ setuptools-rust -cryptography==3.2 +cryptography==3.3.2 wheel requests awscli @@ -11,7 +11,6 @@ PyYAML requests ansible appdirs -botocore chardet Cheetah3 colorama diff --git a/router/start.sh b/router/start.sh index ad5d43f..f0b6b19 100644 --- a/router/start.sh +++ b/router/start.sh @@ -6,6 +6,6 @@ unzip -d /etc/openvpn/ /etc/openvpn/openvpn.zip service openvpn start -python ictf-tcpdump.py & +python3 ictf-tcpdump.py & -python ictf-pcap-s3.py \ No newline at end of file +python3 ictf-pcap-s3.py diff --git a/scoreboard/frontend/package.json b/scoreboard/frontend/package.json index 91617bd..2fe7526 100644 --- a/scoreboard/frontend/package.json +++ b/scoreboard/frontend/package.json @@ -19,7 +19,7 @@ "eslint": "^7.32.0", "eslint-plugin-react-hooks": "^4.2.1-alpha-46a0f050a-20210828", "import-glob-loader": "^1.1.0", - "node-sass": "^6.0.1", + "node-sass": "^7.0.0", "react": "^17.0.2", "react-addons-css-transition-group": "^15.6.2", "react-dom": "^17.0.2", diff --git a/scoring_ictf/scoring_ictf/scoring_interface.py b/scoring_ictf/scoring_ictf/scoring_interface.py index 9a6a8c7..9307cc7 100644 --- a/scoring_ictf/scoring_ictf/scoring_interface.py +++ b/scoring_ictf/scoring_ictf/scoring_interface.py @@ -63,7 +63,10 @@ def _get_scores_for_tick(self, tick): self.log.debug("Tick 0 requested, providing default value!") # this is treated as immutable, so one instance is fine :) default = self._default_value() - self._total_scores[0] = {tid: default for tid in self.gsi.team_id_to_name_map.keys()} + if self.gsi.team_id_to_name_map == None: + self._total_scores[0] = {} + else: + self._total_scores[0] = {tid: default for tid in self.gsi.team_id_to_name_map.keys()} if tick not in self._total_scores: t1 = datetime.now() diff --git a/teaminterface/requirements.txt b/teaminterface/requirements.txt index 612f94b..cc4526d 100644 --- a/teaminterface/requirements.txt +++ b/teaminterface/requirements.txt @@ -4,9 +4,9 @@ Flask-HTTPAuth==2.7.0 Flask-RESTful==0.3.4 Flask==0.10.1 GeoIP -Jinja2==2.8 +Jinja2==2.11.3 Pillow==3.4.1 -Werkzeug==0.10.4 +Werkzeug==0.15.3 aniso8601==1.0.0 argparse==1.2.1 backports.ssl-match-hostname==3.4.0.2 From 757413554e1e9e5493636cfac84db1d5af795e25 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Mon, 22 Aug 2022 22:15:23 +0000 Subject: [PATCH 2/2] Fix old python2 code --- gamebot/scripts_facade.py | 2 +- logs_build_infra.txt | 325 +++++++++++++++++++++++--------------- 2 files changed, 197 insertions(+), 130 deletions(-) diff --git a/gamebot/scripts_facade.py b/gamebot/scripts_facade.py index a4945a5..4e2f672 100644 --- a/gamebot/scripts_facade.py +++ b/gamebot/scripts_facade.py @@ -224,7 +224,7 @@ def update_scripts_to_run(self, tick_id, num_benign, num_exploit, num_get_flags) benign_scripts.extend(list(random.choice(non_exploit_scripts[curr_service]["benign"]) for _ in range(num_benign))) # Get all exploit scripts for this service, not submitted from current team - curr_service_exploit_script.extend(value[curr_service]["exploit"] for key, value in exploit_scripts.iteritems() + curr_service_exploit_script.extend(value[curr_service]["exploit"] for key, value in exploit_scripts.items() if key != curr_team and (curr_service in value)) # Flatten the list curr_service_exploit_script = flatten_list(curr_service_exploit_script) diff --git a/logs_build_infra.txt b/logs_build_infra.txt index ec74157..42c9e2f 100644 --- a/logs_build_infra.txt +++ b/logs_build_infra.txt @@ -21,148 +21,99 @@ Step 7/7 : RUN ansible-playbook /root/ansible-provisioning.yml Successfully built 0cabdeab89dc Successfully tagged ictf_base:latest -Step 1/6 : FROM rabbitmq:management - ---> 321a17c29307 -Step 2/6 : ENV RABBITMQ_PID_FILE /var/lib/rabbitmq/mnesia/rabbitmq - ---> Using cache - ---> 2655892c5868 -Step 3/6 : COPY ./dispatcher /opt/ictf/dispatcher - ---> Using cache - ---> beb9a1b2cee0 -Step 4/6 : WORKDIR /opt/ictf/dispatcher - ---> Using cache - ---> 21b22e96aee7 -Step 5/6 : RUN chmod +x ./start.sh - ---> Using cache - ---> 99a175e979fc -Step 6/6 : ENTRYPOINT ./start.sh - ---> Using cache - ---> 26364e2a1a60 - -Successfully built 26364e2a1a60 -Step 1/9 : FROM ictf_base - ---> 0cabdeab89dc -Step 2/9 : RUN DEBIAN_FRONTEND=noninteractive apt-get install -y python3-pip python3-virtualenv python3 python3-dev git libssl-dev libffi-dev build-essential libc6-dev-i386 - ---> Using cache - ---> 82239dba4157 -Step 3/9 : COPY ./scriptbot /opt/ictf/scriptbot - ---> Using cache - ---> 9d9db15acf4a -Step 4/9 : WORKDIR /opt/ictf/scriptbot - ---> Using cache - ---> c005b43f5cd8 -Step 5/9 : COPY ./common/hephaestus_provisioning/teamhosts ./teamhosts - ---> Using cache - ---> cc5fcb075bf2 -Step 6/9 : RUN pip3 install -r requirements.txt - ---> Using cache - ---> 34016535702f -Step 7/9 : RUN ansible-playbook provisioning/hephaestus_provisioning/ansible-provisioning.yml - ---> Using cache - ---> cacc08817262 -Step 8/9 : RUN chmod +x ./start.sh && chmod +x ./start_idle_mode.sh - ---> Using cache - ---> 76f21420d0c9 -Step 9/9 : ENTRYPOINT ./start.sh - ---> Using cache - ---> d998ad48c56d - -Successfully tagged ictf_dispatcher:latest Step 1/8 : FROM ictf_base - ---> 0cabdeab89dc -Step 2/8 : RUN DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends python3-pip python3-wheel - ---> Using cache - ---> d096f32f65c1 -Step 3/8 : COPY ./gamebot /opt/ictf/gamebot - ---> Using cache - ---> c592e46a9638 -Step 4/8 : WORKDIR /opt/ictf/gamebot - ---> Using cache - ---> 5b1dd5b61645 -Step 5/8 : RUN pip install -r requirements.txt - ---> Using cache - ---> ffdc3d4b806a -Step 6/8 : RUN chmod +x ./start.sh - ---> Using cache - ---> fbf83d307a12 -Step 7/8 : RUN ansible-playbook ./provisioning/hephaestus_provisioning/ansible-provisioning.yml --extra-vars ICTF_API_ADDRESS="database.ictf" - ---> Using cache - ---> 2c5e97fbd9c6 -Step 8/8 : ENTRYPOINT ./start.sh - ---> Using cache - ---> 6c681b1a486a - Step 1/13 : FROM ictf_base ---> 0cabdeab89dc +Step 2/8 : RUN DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends iptraf python3 python3-pip python3-dev build-essential awscli iptables-persistent iftop libcurl4-openssl-dev libssl-dev python3-apt zip unzip openvpn python3-setuptools python3-wheel + ---> 0cabdeab89dc Step 2/13 : ENV DEBIAN_FRONTEND=noninteractive + ---> Using cache + ---> b196a60c4431 +Step 3/8 : COPY ./router /opt/ictf/router ---> Using cache ---> 2f9d82f3ffd9 Step 3/13 : RUN apt-get update && apt-get install -y nginx git redis-server build-essential + ---> Using cache + ---> 9b99adbbfc25 +Step 4/8 : WORKDIR /opt/ictf/router ---> Using cache ---> 91355d09b17d Step 4/13 : RUN apt-get update && apt-get install -y sudo uwsgi daemon ---> Using cache ---> 9b5578c6dc5e Step 5/13 : RUN curl -sL https://deb.nodesource.com/setup_14.x | bash - + ---> Using cache + ---> 809563d1b92a +Step 5/8 : RUN pip install -r requirements3.txt ---> Using cache ---> a124d9ec37aa Step 6/13 : RUN apt-get update && apt-get install -y nodejs + ---> Using cache + ---> d898a1a95858 +Step 6/8 : RUN ansible-playbook provisioning/hephaestus_provisioning/ansible-provisioning.yml --become ---> Using cache ---> 0b54ac056290 Step 7/13 : RUN apt-get update && apt-get install -y python3-pip python3-virtualenv python3-dev uwsgi-plugin-python3 python3-setuptools python3-wheel + ---> Using cache + ---> 817aeda275ce +Step 7/8 : RUN chmod +x ./start.sh ---> Using cache ---> 69f6466cf7df Step 8/13 : COPY ./scoreboard /opt/ictf/scoreboard + ---> Using cache + ---> 06d316ec2e21 +Step 8/8 : ENTRYPOINT ./start.sh + ---> Using cache + ---> 6726981fb9f8 + ---> Using cache ---> 478f66ea1162 Step 9/13 : WORKDIR /opt/ictf/scoreboard ---> Using cache ---> c0ba46f898b6 Step 10/13 : RUN pip install -r requirements.txt +Step 1/8 : FROM ictf_base + ---> 0cabdeab89dc +Step 2/8 : RUN DEBIAN_FRONTEND=noninteractive apt-get install -y nginx python3-pip python3-virtualenv python3-dev libjpeg8 libjpeg-dev zlib1g zlib1g-dev libpng-dev libmemcached-dev libmemcached-tools libgeoip1 libgeoip-dev geoip-bin geoip-database redis-server + ---> Using cache + ---> 6fcde0df0a58 ---> Using cache ---> 1e5140cf78b1 Step 11/13 : RUN ansible-playbook provisioning/hephaestus_provisioning/ansible-provisioning.yml --become --extra-vars ICTF_USER="root" --extra-vars ICTF_FRAMEWORK_DIR_HOST="/opt/ictf" --extra-vars ICTF_API_ADDRESS="database.ictf" +Step 3/8 : COPY ./teaminterface /opt/ictf/teaminterface ---> Using cache ---> 9ed67e11f8c0 Step 12/13 : RUN chmod +x ./start.sh +Step 1/9 : FROM ictf_base + ---> 0cabdeab89dc ---> Using cache ---> fa2097412a8f Step 13/13 : ENTRYPOINT ./start.sh +Step 2/9 : RUN DEBIAN_FRONTEND=noninteractive apt-get install -y python3-pip python3-virtualenv python3 python3-dev git libssl-dev libffi-dev build-essential libc6-dev-i386 ---> Using cache - ---> a65b12846144 - -Successfully built d998ad48c56d -Successfully tagged ictf_scriptbot:latest -Successfully built 6c681b1a486a -Successfully built a65b12846144 -Successfully tagged ictf_gamebot:latest -Successfully tagged ictf_scoreboard:latest -Step 1/8 : FROM ictf_base - ---> 0cabdeab89dc -Step 2/8 : RUN DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends iptraf python3 python3-pip python3-dev build-essential awscli iptables-persistent iftop libcurl4-openssl-dev libssl-dev python3-apt zip unzip openvpn python3-setuptools python3-wheel + ---> 82239dba4157 +Step 3/9 : COPY ./scriptbot /opt/ictf/scriptbot ---> Using cache - ---> b196a60c4431 -Step 3/8 : COPY ./router /opt/ictf/router + ---> 9d9db15acf4a +Step 4/9 : WORKDIR /opt/ictf/scriptbot ---> Using cache - ---> 9b99adbbfc25 -Step 4/8 : WORKDIR /opt/ictf/router + ---> c005b43f5cd8 +Step 5/9 : COPY ./common/hephaestus_provisioning/teamhosts ./teamhosts ---> Using cache - ---> 809563d1b92a -Step 5/8 : RUN pip install -r requirements3.txt + ---> f3621945ca6c +Step 4/8 : WORKDIR /opt/ictf/teaminterface ---> Using cache - ---> d898a1a95858 -Step 6/8 : RUN ansible-playbook provisioning/hephaestus_provisioning/ansible-provisioning.yml --become + ---> cc01211f96da +Step 5/8 : RUN pip install -r requirements.txt ---> Using cache - ---> 817aeda275ce -Step 7/8 : RUN chmod +x ./start.sh + ---> a65b12846144 + ---> Using cache - ---> 06d316ec2e21 -Step 8/8 : ENTRYPOINT ./start.sh + ---> cc5fcb075bf2 +Step 6/9 : RUN pip3 install -r requirements.txt ---> Using cache - ---> 6726981fb9f8 - -Successfully built 6726981fb9f8 -Successfully tagged ictf_router:latest + ---> fe28c9ff298c +Step 6/8 : RUN ansible-playbook provisioning/hephaestus_provisioning/ansible-provisioning.yml --become --extra-vars ICTF_DB_API_ADDRESS="database.ictf" Step 1/8 : FROM ictf_base ---> 0cabdeab89dc Step 2/8 : RUN DEBIAN_FRONTEND=noninteractive apt-get install -y nginx uwsgi uwsgi-plugin-python3 mysql-server python3-pip python3-virtualenv python3-mysqldb python3-dev python3-setuptools python3-wheel cron daemon @@ -173,28 +124,81 @@ Step 3/8 : RUN pip install --upgrade pip ---> fca8b017f7b8 Step 4/8 : COPY ./database /opt/ictf/database ---> Using cache - ---> a4b800b1dd80 + ---> 33f926123590 Step 5/8 : COPY ./scoring_ictf /opt/ictf/scoring_ictf ---> Using cache - ---> 00073f2844f6 + ---> 34016535702f +Step 7/9 : RUN ansible-playbook provisioning/hephaestus_provisioning/ansible-provisioning.yml + ---> Using cache + ---> c6cd00b82763 +Step 7/8 : RUN chmod +x ./start.sh + ---> Using cache + ---> 2c7c731c879b Step 6/8 : WORKDIR /opt/ictf/database ---> Using cache - ---> 85ef25e9720d + ---> cacc08817262 +Step 8/9 : RUN chmod +x ./start.sh && chmod +x ./start_idle_mode.sh + ---> Using cache + ---> Using cache + ---> 92e9744ef0a2 +Step 8/8 : ENTRYPOINT ./start.sh + ---> 5f62ae3742ef Step 7/8 : RUN chmod +x ./start.sh && ansible-playbook ./provisioning/hephaestus_provisioning/ansible-provisioning.yml ---> Using cache - ---> 775cd2e600bf + ---> 76f21420d0c9 +Step 9/9 : ENTRYPOINT ./start.sh + ---> Using cache + ---> e2cda82891b0 + + ---> Using cache + ---> c2ad0aea0f9f Step 8/8 : ENTRYPOINT ./start.sh ---> Using cache - ---> 99c4d513a85b + ---> d998ad48c56d + + ---> Using cache + ---> 8287accb475c -Successfully built 99c4d513a85b +Successfully built 6726981fb9f8 +Successfully tagged ictf_router:latest +Successfully built a65b12846144 +Successfully built e2cda82891b0 +Successfully built d998ad48c56d +Successfully tagged ictf_teaminterface:latest +Successfully tagged ictf_scoreboard:latest +Successfully tagged ictf_scriptbot:latest +Successfully built 8287accb475c Successfully tagged ictf_database:latest -Step 1/8 : FROM ubuntu:18.04 +Step 1/8 : FROM ubuntu:18.04Step 1/6 : FROM rabbitmq:management + ---> 321a17c29307 +Step 2/6 : ENV RABBITMQ_PID_FILE /var/lib/rabbitmq/mnesia/rabbitmq + ---> Using cache + ---> 2655892c5868 +Step 3/6 : COPY ./dispatcher /opt/ictf/dispatcher +Step 1/8 : FROM ictf_base + ---> 8d5df41c547b Step 2/8 : RUN DEBIAN_FRONTEND=noninteractive apt-get -y --no-install-recommends -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" upgrade && apt-get update -y && apt-get -y install software-properties-common && apt-add-repository ppa:ansible/ansible && apt-get -y update && apt -y install ansible git wget curl vim iputils-ping python-pip python-dev python3-dev build-essential htop + ---> 0cabdeab89dc +Step 2/8 : RUN DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends python3-pip python3-wheel + ---> Using cache + ---> beb9a1b2cee0 +Step 4/6 : WORKDIR /opt/ictf/dispatcher + ---> Using cache + ---> 21b22e96aee7 +Step 5/6 : RUN chmod +x ./start.sh + ---> Using cache + ---> 99a175e979fc +Step 6/6 : ENTRYPOINT ./start.sh ---> Using cache ---> 7b4045115698 Step 3/8 : COPY ./secrets /opt/ictf/secrets + ---> Using cache + ---> 26364e2a1a60 + + ---> Using cache + ---> d096f32f65c1 +Step 3/8 : COPY ./gamebot /opt/ictf/gamebot ---> Using cache ---> 87cc403ef32f Step 4/8 : COPY ./logger /opt/ictf/logger @@ -213,32 +217,95 @@ Step 8/8 : ENTRYPOINT ./start.sh ---> Using cache ---> 554f7987b11b -Step 1/8 : FROM ictf_base - ---> 0cabdeab89dc -Step 2/8 : RUN DEBIAN_FRONTEND=noninteractive apt-get install -y nginx python3-pip python3-virtualenv python3-dev libjpeg8 libjpeg-dev zlib1g zlib1g-dev libpng-dev libmemcached-dev libmemcached-tools libgeoip1 libgeoip-dev geoip-bin geoip-database redis-server - ---> Using cache - ---> 6fcde0df0a58 -Step 3/8 : COPY ./teaminterface /opt/ictf/teaminterface - ---> Using cache - ---> f3621945ca6c -Step 4/8 : WORKDIR /opt/ictf/teaminterface - ---> Using cache - ---> cc01211f96da +Successfully built 26364e2a1a60 +Successfully tagged ictf_dispatcher:latest +Successfully built 554f7987b11b +Successfully tagged ictf_logger:latest + ---> a9643e1f621c +Step 4/8 : WORKDIR /opt/ictf/gamebot + ---> Running in e8b382e93bab +Removing intermediate container e8b382e93bab + ---> cf6f138a8b86 Step 5/8 : RUN pip install -r requirements.txt - ---> Using cache - ---> fe28c9ff298c -Step 6/8 : RUN ansible-playbook provisioning/hephaestus_provisioning/ansible-provisioning.yml --become --extra-vars ICTF_DB_API_ADDRESS="database.ictf" - ---> Using cache - ---> c6cd00b82763 -Step 7/8 : RUN chmod +x ./start.sh - ---> Using cache - ---> 92e9744ef0a2 + ---> Running in 71ae1c8cc68c +Requirement already satisfied: wheel in /usr/lib/python3/dist-packages (from -r requirements.txt (line 1)) (0.34.2) +Collecting coloredlogs==7.3.1 + Downloading coloredlogs-7.3.1-py2.py3-none-any.whl (37 kB) +Collecting requests==2.18.4 + Downloading requests-2.18.4-py2.py3-none-any.whl (88 kB) +Collecting setuptools==44.0.0 + Downloading setuptools-44.0.0-py2.py3-none-any.whl (583 kB) +Collecting python-logstash + Downloading python-logstash-0.4.8.tar.gz (7.1 kB) +Collecting pika + Downloading pika-1.3.0-py3-none-any.whl (155 kB) +Collecting humanfriendly>=3.2 + Downloading humanfriendly-10.0-py2.py3-none-any.whl (86 kB) +Collecting idna<2.7,>=2.5 + Downloading idna-2.6-py2.py3-none-any.whl (56 kB) +Requirement already satisfied: chardet<3.1.0,>=3.0.2 in /usr/lib/python3/dist-packages (from requests==2.18.4->-r requirements.txt (line 3)) (3.0.4) +Collecting urllib3<1.23,>=1.21.1 + Downloading urllib3-1.22-py2.py3-none-any.whl (132 kB) +Requirement already satisfied: certifi>=2017.4.17 in /usr/lib/python3/dist-packages (from requests==2.18.4->-r requirements.txt (line 3)) (2019.11.28) +Building wheels for collected packages: python-logstash + Building wheel for python-logstash (setup.py): started + Building wheel for python-logstash (setup.py): finished with status 'done' + Created wheel for python-logstash: filename=python_logstash-0.4.8-py3-none-any.whl size=8415 sha256=5ba973ce639dde1f044b74c9f3ddad851b74885ae16afeccbdeae06f4641a424 + Stored in directory: /root/.cache/pip/wheels/eb/e4/85/edb12495360894b52ab9161f28d348250b41095ee4a322c16e +Successfully built python-logstash +Installing collected packages: humanfriendly, coloredlogs, idna, urllib3, requests, setuptools, python-logstash, pika + Attempting uninstall: idna + Found existing installation: idna 2.8 + Not uninstalling idna at /usr/lib/python3/dist-packages, outside environment /usr + Can't uninstall 'idna'. No files were found to uninstall. + Attempting uninstall: urllib3 + Found existing installation: urllib3 1.25.8 + Not uninstalling urllib3 at /usr/lib/python3/dist-packages, outside environment /usr + Can't uninstall 'urllib3'. No files were found to uninstall. + Attempting uninstall: requests + Found existing installation: requests 2.22.0 + Not uninstalling requests at /usr/lib/python3/dist-packages, outside environment /usr + Can't uninstall 'requests'. No files were found to uninstall. + Attempting uninstall: setuptools + Found existing installation: setuptools 45.2.0 + Not uninstalling setuptools at /usr/lib/python3/dist-packages, outside environment /usr + Can't uninstall 'setuptools'. No files were found to uninstall. +Successfully installed coloredlogs-7.3.1 humanfriendly-10.0 idna-2.6 pika-1.3.0 python-logstash-0.4.8 requests-2.18.4 setuptools-44.0.0 urllib3-1.22 +Removing intermediate container 71ae1c8cc68c + ---> a7ad1493259f +Step 6/8 : RUN chmod +x ./start.sh + ---> Running in d22703acc5f5 +Removing intermediate container d22703acc5f5 + ---> 50283d77377c +Step 7/8 : RUN ansible-playbook ./provisioning/hephaestus_provisioning/ansible-provisioning.yml --extra-vars ICTF_API_ADDRESS="database.ictf" + ---> Running in 62631a2a4af9 +[WARNING]: provided hosts list is empty, only localhost is available. Note that +the implicit localhost does not match 'all' + +PLAY [localhost] *************************************************************** + +TASK [Gathering Facts] ********************************************************* +ok: [localhost] + +TASK [ictf-database database connection settings.py configuration - set ICTF_API_ADDRESS] *** +changed: [localhost] + +TASK [ictf-database database connection config.json configuration - set API_SECRET] *** +changed: [localhost] + +TASK [logstash - config] ******************************************************* +changed: [localhost] + +PLAY RECAP ********************************************************************* +localhost : ok=4 changed=3 unreachable=0 failed=0 skipped=0 rescued=0 ignored=0 + +Removing intermediate container 62631a2a4af9 + ---> 90e28801a83f Step 8/8 : ENTRYPOINT ./start.sh - ---> Using cache - ---> e2cda82891b0 + ---> Running in 16513a208fa4 +Removing intermediate container 16513a208fa4 + ---> 7e6bd0eace41 -Successfully built 554f7987b11b -Successfully tagged ictf_logger:latest -Successfully built e2cda82891b0 -Successfully tagged ictf_teaminterface:latest +Successfully built 7e6bd0eace41 +Successfully tagged ictf_gamebot:latest /home/ubuntu/ictf-framework