Skip to content

Commit 3e33522

Browse files
bertskybertsky
authored
Merge pull request #452 from OCR-D/update-2024-10-11
Update 2024 10 11
2 parents 25fa279 + e33ac77 commit 3e33522

25 files changed

+645
-92
lines changed

‎.circleci/config.yml

Lines changed: 244 additions & 55 deletions
Original file line numberDiff line numberDiff line change
@@ -1,78 +1,109 @@
11
version: 2.1
2-
jobs:
2+
3+
commands:
4+
dockerhub-store:
5+
description: "store an image on Dockerhub"
6+
parameters:
7+
tag:
8+
description: "image tag (full variant name) to be saved"
9+
type:
10+
string
11+
steps:
12+
- run:
13+
name: Login to Docker Hub
14+
command: echo "$DOCKERHUB_PASS" | docker login --username "$DOCKERHUB_USERNAME" --password-stdin
15+
- run:
16+
name: save image
17+
command: |
18+
docker tag ocrd/all:<< parameters.tag >> ocrd/all:<< parameters.tag >>-${CIRCLE_SHA1}
19+
docker push ocrd/all:<< parameters.tag >>-${CIRCLE_SHA1}
20+
no_output_timeout: 30m
21+
dockerhub-load:
22+
description: "load an image from Dockerhub"
23+
parameters:
24+
tag:
25+
description: "image tag (full variant name) to be pulled"
26+
type:
27+
string
28+
steps:
29+
- run:
30+
name: load previous image
31+
command: |
32+
docker pull ocrd/all:<< parameters.tag >>-${CIRCLE_SHA1}
33+
docker tag ocrd/all:<< parameters.tag >>-${CIRCLE_SHA1} ocrd/all:<< parameters.tag >>
34+
dockerhub-remove:
35+
description: "remove an image on Dockerhub"
36+
parameters:
37+
tag:
38+
description: "image tag (full variant name) to be removed"
39+
type:
40+
string
41+
steps:
42+
- run:
43+
name: use Dockerhub API to remove temporary image
44+
command: |
45+
HUB_TOKEN=$(curl -s -H "Content-Type: application/json" -X POST -d "{\"username\": \"$DOCKERHUB_USERNAME\", \"password\": \"$DOCKERHUB_PASS\"}" https://hub.docker.com/v2/users/login/ | jq -r .token)
46+
set -x
47+
curl -i -X DELETE -H "Accept: application/json" -H "Authorization: JWT $HUB_TOKEN" https://hub.docker.com/v2/namespaces/ocrd/repositories/all/tags/<< parameters.tag >>-${CIRCLE_SHA1}/
48+
# must be unconditional (both on_error and on_success)
49+
when: always
350
build:
4-
docker:
5-
- image: cimg/base:current-22.04
6-
resource_class: large
51+
description: "template for all docker-* build jobs"
52+
parameters:
53+
tag:
54+
description: "image tag (full variant name) to be built"
55+
type:
56+
string
57+
dep:
58+
description: "image tag (full variant name) this depends on (should _not_ be rebuilt)"
59+
type:
60+
string
61+
default: "core"
762
steps:
863
- checkout
9-
- setup_remote_docker # https://circleci.com/docs/2.0/building-docker-images/
64+
- setup_remote_docker
1065
- run:
11-
name: build image
12-
command: make docker-maximum-cuda GIT_DEPTH=--single-branch DOCKER_PARALLEL=-j4
66+
name: Build Docker image
67+
command: make docker-<< parameters.tag >> -o docker-<< parameters.dep >> GIT_DEPTH=--single-branch
1368
no_output_timeout: 30m
1469
- run:
1570
name: test image
1671
command: |
1772
mkdir test-results
18-
docker run --rm -v $PWD:/data ocrd/all:maximum-cuda make -C /build/core deps-test test PYTEST_ARGS=--junitxml=/data/test-results/core.xml
73+
# cannot use docker run -v because the docker executor does not support volumes
74+
# docker run --rm -v $PWD:/data ocrd/all:<< parameters.tag >> make -C /build/core deps-test test PYTEST_ARGS=--junitxml=/data/test-results/core.xml
75+
id=`docker create ocrd/all:<< parameters.tag >> make -C /build/core deps-test test PYTEST_ARGS=--junitxml=core.xml`
76+
docker start -a $id
77+
docker cp $id:/build/core/core.xml test-results/
78+
docker rm $id
1979
- store_test_results:
20-
path: test-results
21-
- when:
22-
# takes too long for 1h1m CircleCI timeout overall
23-
# also, storage is limited...
24-
condition: false
25-
steps:
26-
- run:
27-
name: persist image
28-
command: |
29-
sudo apt install pigz
30-
docker image save ocrd/all:maximum-cuda | pigz --fast > ocrd-all-maximum.tar.gz
31-
no_output_timeout: 30m
32-
# can be downloaded from CircleCI.com and imported via "docker image load"
33-
- store_artifacts:
34-
path: ocrd-all-maximum.tar.gz
35-
destination: artifacts
80+
path: test-results/core.xml
3681
deploy:
37-
docker:
38-
- image: cimg/base:current-22.04
82+
description: "template for all docker-* deploy jobs"
3983
parameters:
40-
variant:
84+
tag:
85+
description: "image tag (full variant name) to be pushed"
4186
type:
4287
string
4388
steps:
4489
- checkout
45-
- setup_remote_docker # https://circleci.com/docs/2.0/building-docker-images/
90+
- setup_remote_docker
4691
- run:
47-
name: Build Docker image
48-
command: make docker-<< parameters.variant >> GIT_DEPTH=--single-branch
49-
# fails due to pip races: DOCKER_PARALLEL=-j3
50-
no_output_timeout: 30m
51-
- run:
52-
name: Alias Docker images
53-
command: docker tag ocrd/all:<< parameters.variant >> ocrd/all:<< parameters.variant >>-git
92+
name: Alias Docker image
93+
command: docker tag ocrd/all:<< parameters.tag >> ocrd/all:<< parameters.tag >>-git
5494
- run:
5595
name: Login to Docker Hub
5696
command: echo "$DOCKERHUB_PASS" | docker login --username "$DOCKERHUB_USERNAME" --password-stdin
5797
- run:
5898
name: Push images to Docker Hub
5999
no_output_timeout: 2.5h
60100
command: |
61-
docker push ocrd/all:<< parameters.variant >>
62-
docker push ocrd/all:<< parameters.variant >>-git
101+
docker push ocrd/all:<< parameters.tag >>
102+
docker push ocrd/all:<< parameters.tag >>-git
63103
- when:
64104
condition:
65-
equal: [ maximum, << parameters.variant >> ]
105+
equal: [ maximum, << parameters.tag >> ]
66106
steps:
67-
- run:
68-
name: Alias and push intermediate variants
69-
command: |
70-
docker tag ocrd/all:medium ocrd/all:medium-git
71-
docker tag ocrd/all:minimum ocrd/all:minimum-git
72-
docker push ocrd/all:minimum
73-
docker push ocrd/all:minimum-git
74-
docker push ocrd/all:medium
75-
docker push ocrd/all:medium-git
76107
- run:
77108
name: store ocrd-all-tool.json
78109
command: |
@@ -91,28 +122,186 @@ jobs:
91122
destination: ocrd-all-module-dir.json
92123
- when:
93124
condition:
94-
equal: [ maximum-cuda, << parameters.variant >> ]
125+
equal: [ maximum-cuda, << parameters.tag >> ]
95126
steps:
96127
- run:
97128
name: Create a date-versioned mirror of ocrd/all:maximum-cuda
98129
command: bash release.sh release-dockerhub
99130
- run:
100131
name: Update badge
101132
command: curl -X POST "$MICROBADGER_WEBHOOK" || true
133+
- when:
134+
# takes too long for 1h1m CircleCI timeout overall
135+
# also, storage is limited...
136+
condition: false
137+
steps:
138+
- run:
139+
name: persist image
140+
command: |
141+
sudo apt install pigz
142+
docker image save ocrd/all:<< parameters.tag >> | pigz --fast > ocrd-all-maximum.tar.gz
143+
no_output_timeout: 30m
144+
# can be downloaded from CircleCI.com and imported via "docker image load"
145+
- store_artifacts:
146+
path: ocrd-all-maximum.tar.gz
147+
destination: artifacts
148+
149+
jobs:
150+
build-mini:
151+
docker:
152+
- image: cimg/base:current-22.04
153+
parameters:
154+
variant:
155+
type:
156+
string
157+
steps:
158+
- build:
159+
tag: minim<< parameters.variant >>
160+
- dockerhub-store:
161+
tag: minim<< parameters.variant >>
162+
build-medi:
163+
docker:
164+
- image: cimg/base:current-22.04
165+
parameters:
166+
variant:
167+
type:
168+
string
169+
steps:
170+
- dockerhub-load:
171+
tag: minim<< parameters.variant >>
172+
- unless:
173+
condition:
174+
equal: [ master, << pipeline.git.branch >>]
175+
# only PR testing – not needed anymore
176+
steps:
177+
- dockerhub-remove:
178+
tag: minim<< parameters.variant >>
179+
- build:
180+
dep: minim<< parameters.variant >>
181+
tag: medi<< parameters.variant >>
182+
- dockerhub-store:
183+
tag: medi<< parameters.variant >>
184+
build-maxi:
185+
docker:
186+
- image: cimg/base:current-22.04
187+
parameters:
188+
variant:
189+
type:
190+
string
191+
steps:
192+
- dockerhub-load:
193+
tag: medi<< parameters.variant >>
194+
- unless:
195+
condition:
196+
equal: [ master, << pipeline.git.branch >>]
197+
# only PR testing – not needed anymore
198+
steps:
199+
- dockerhub-remove:
200+
tag: medi<< parameters.variant >>
201+
- build:
202+
dep: medi<< parameters.variant >>
203+
tag: maxim<< parameters.variant >>
204+
- when:
205+
condition:
206+
equal: [ master, << pipeline.git.branch >>]
207+
# only PR testing – not needed
208+
steps:
209+
- dockerhub-store:
210+
tag: maxim<< parameters.variant >>
211+
deploy-mini:
212+
docker:
213+
- image: cimg/base:current-22.04
214+
parameters:
215+
variant:
216+
type:
217+
string
218+
steps:
219+
- dockerhub-load:
220+
tag: minim<< parameters.variant >>
221+
- deploy:
222+
tag: minim<< parameters.variant >>
223+
- dockerhub-remove:
224+
tag: minim<< parameters.variant >>
225+
deploy-medi:
226+
docker:
227+
- image: cimg/base:current-22.04
228+
parameters:
229+
variant:
230+
type:
231+
string
232+
steps:
233+
- dockerhub-load:
234+
tag: medi<< parameters.variant >>
235+
- deploy:
236+
tag: medi<< parameters.variant >>
237+
- dockerhub-remove:
238+
tag: medi<< parameters.variant >>
239+
deploy-maxi:
240+
docker:
241+
- image: cimg/base:current-22.04
242+
parameters:
243+
variant:
244+
type:
245+
string
246+
steps:
247+
- dockerhub-load:
248+
tag: maxim<< parameters.variant >>
249+
- deploy:
250+
tag: maxim<< parameters.variant >>
251+
- dockerhub-remove:
252+
tag: maxim<< parameters.variant >>
253+
102254
workflows:
103255
version: 2
104-
build-master:
256+
build:
105257
jobs:
106-
- deploy:
258+
- build-mini:
259+
name: build-minim<< matrix.variant >>
260+
matrix:
261+
parameters:
262+
variant: [um, um-cuda]
263+
- build-medi:
264+
name: build-medi<< matrix.variant >>
265+
matrix:
266+
parameters:
267+
variant: [um, um-cuda]
268+
requires:
269+
- build-minim<< matrix.variant >>
270+
- build-maxi:
271+
name: build-maxim<< matrix.variant >>
107272
matrix:
108273
parameters:
109-
variant: [maximum, maximum-cuda]
274+
variant: [um, um-cuda]
275+
requires:
276+
- build-medi<< matrix.variant >>
277+
- deploy-mini:
278+
name: deploy-minim<< matrix.variant >>
279+
matrix:
280+
parameters:
281+
variant: [um, um-cuda]
282+
requires:
283+
- build-minim<< matrix.variant >>
110284
filters:
111285
branches:
112286
only: master
113-
build-pull-request:
114-
jobs:
115-
- build:
287+
- deploy-medi:
288+
name: deploy-medi<< matrix.variant >>
289+
matrix:
290+
parameters:
291+
variant: [um, um-cuda]
292+
requires:
293+
- build-medi<< matrix.variant >>
294+
filters:
295+
branches:
296+
only: master
297+
- deploy-maxi:
298+
name: deploy-maxim<< matrix.variant >>
299+
matrix:
300+
parameters:
301+
variant: [um, um-cuda]
302+
requires:
303+
- build-maxim<< matrix.variant >>
116304
filters:
117305
branches:
118-
ignore: master
306+
only: master
307+

‎.gitmodules

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -94,3 +94,6 @@
9494
[submodule "ocrd_froc"]
9595
path = ocrd_froc
9696
url = https://github.com/OCR-D/ocrd_froc
97+
[submodule "ocrd_page2alto"]
98+
path = ocrd_page2alto
99+
url = https://github.com/OCR-D/page-to-alto

0 commit comments

Comments
 (0)