Skip to content

Commit

Permalink
Refine integration test
Browse files Browse the repository at this point in the history
  • Loading branch information
sevein committed May 26, 2024
1 parent a2780cb commit 271a9a4
Show file tree
Hide file tree
Showing 16 changed files with 960 additions and 320 deletions.
5 changes: 3 additions & 2 deletions hack/ccp/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,10 @@ require (
connectrpc.com/connect v1.16.2
connectrpc.com/grpchealth v1.3.0
connectrpc.com/grpcreflect v1.2.0
github.com/artefactual-labs/gearmin v0.0.0-20240507145739-e15b2dbf710c
github.com/artefactual-labs/gearmin v0.0.0-20240525053553-51122ec3bbfd
github.com/bufbuild/protovalidate-go v0.6.2
github.com/cenkalti/backoff/v4 v4.3.0
github.com/docker/docker v25.0.5+incompatible
github.com/doug-martin/goqu/v9 v9.19.0
github.com/elliotchance/orderedmap/v2 v2.2.0
github.com/fsnotify/fsnotify v1.7.0
Expand All @@ -19,6 +20,7 @@ require (
github.com/google/uuid v1.6.0
github.com/hashicorp/go-retryablehttp v0.7.6
github.com/mikespook/gearman-go v0.0.0-20220520031403-2a518e866145
github.com/otiai10/copy v1.14.0
github.com/peterbourgon/ff/v3 v3.4.0
github.com/rs/cors v1.11.0
github.com/tailscale/hujson v0.0.0-20221223112325-20486734a56a
Expand Down Expand Up @@ -55,7 +57,6 @@ require (
github.com/cpuguy83/dockercfg v0.3.1 // indirect
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
github.com/distribution/reference v0.5.0 // indirect
github.com/docker/docker v25.0.5+incompatible // indirect
github.com/docker/go-connections v0.5.0 // indirect
github.com/docker/go-units v0.5.0 // indirect
github.com/felixge/httpsnoop v1.0.4 // indirect
Expand Down
8 changes: 6 additions & 2 deletions hack/ccp/go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -30,8 +30,8 @@ github.com/antlr4-go/antlr/v4 v4.13.0 h1:lxCg3LAv+EUK6t1i0y1V6/SLeUi0eKEKdhQAlS8
github.com/antlr4-go/antlr/v4 v4.13.0/go.mod h1:pfChB/xh/Unjila75QW7+VU4TSnWnnk9UTnmpPaOR2g=
github.com/armon/go-radix v1.0.1-0.20221118154546-54df44f2176c h1:651/eoCRnQ7YtSjAnSzRucrJz+3iGEFt+ysraELS81M=
github.com/armon/go-radix v1.0.1-0.20221118154546-54df44f2176c/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
github.com/artefactual-labs/gearmin v0.0.0-20240507145739-e15b2dbf710c h1:vf3GatdfaJojurMPb5g4V2yujXPGwSQZnEs37crmfcA=
github.com/artefactual-labs/gearmin v0.0.0-20240507145739-e15b2dbf710c/go.mod h1:C+JaJpQwRvWN3sa2z2jy+ziy7mQuuTIsTiRFJbUBO38=
github.com/artefactual-labs/gearmin v0.0.0-20240525053553-51122ec3bbfd h1:b9WEVrgVymFz99oBFbfHuzHvnMlDZDNGlGVUB922Rb8=
github.com/artefactual-labs/gearmin v0.0.0-20240525053553-51122ec3bbfd/go.mod h1:0bctkvvsETfNnBXLrUNvHY8FSxFC372jrIhddQLIyBI=
github.com/bep/clocks v0.5.0 h1:hhvKVGLPQWRVsBP/UB7ErrHYIO42gINVbvqxvYTPVps=
github.com/bep/clocks v0.5.0/go.mod h1:SUq3q+OOq41y2lRQqH5fsOoxN8GbxSiT6jvoVVLCVhU=
github.com/bep/debounce v1.2.0 h1:wXds8Kq8qRfwAOpAxHrJDbCXgC5aHSzgQb/0gKsHQqo=
Expand Down Expand Up @@ -285,6 +285,10 @@ github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8
github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug=
github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM=
github.com/otiai10/copy v1.14.0 h1:dCI/t1iTdYGtkvCuBG2BgR6KZa83PTclw4U5n2wAllU=
github.com/otiai10/copy v1.14.0/go.mod h1:ECfuL02W+/FkTWZWgQqXPWZgW9oeKCSQ5qVfSc4qc4w=
github.com/otiai10/mint v1.5.1 h1:XaPLeE+9vGbuyEHem1JNk3bYc7KKqyI/na0/mLd/Kks=
github.com/otiai10/mint v1.5.1/go.mod h1:MJm72SBthJjz8qhefc4z1PYEieWmy8Bku7CjcAqyUSM=
github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 h1:onHthvaw9LFnH4t2DcNVpwGmV9E1BkGknEliJkfwQj0=
github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhMYhSNPKjeNKa5WY9YCIEBRbNzFFPJbWO6Y=
github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8=
Expand Down
6 changes: 2 additions & 4 deletions hack/ccp/hack/integration.sh
Original file line number Diff line number Diff line change
Expand Up @@ -5,15 +5,13 @@ set -e
readonly __dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
readonly __root="$(cd "$(dirname "${__dir}")" && pwd)"

docker compose stop archivematica-ccp 2>/dev/null

CCP_DIR=${HOME}/.ccp
CCP_AM_DIR=${CCP_DIR}/am-pipeline-data
CCP_SS_DIR=${CCP_DIR}/ss-location-data

env \
CCP_INTEGRATION_ENABLED=1 \
CCP_INTEGRATION_TRANSFER_SOURCE=${CCP_SS_DIR}/archivematica/ \
CCP_INTEGRATION_USE_COMPOSE=yes \
CCP_INTEGRATION_TRANSFER_SOURCE=${CCP_SS_DIR} \
CCP_INTEGRATION_ENABLE_TESTCONTAINERS_LOGGING=yes \
CCP_INTEGRATION_USE_STDOUT=yes \
go test -count=1 -v ${__root}/integration/ -run=TestServerCreatePackage
49 changes: 0 additions & 49 deletions hack/ccp/integration/config_test.go

This file was deleted.

169 changes: 169 additions & 0 deletions hack/ccp/integration/data/Dockerfile.worker
Original file line number Diff line number Diff line change
@@ -0,0 +1,169 @@
# This is a copy of the Archivematica Dockerfile targeting MCPClient only
# I couldn't use the original because testcontainers-go refuses to build it,
# apparently because it's not using BuildKit.

ARG UBUNTU_VERSION=22.04
ARG USER_ID=1000
ARG GROUP_ID=1000
ARG PYTHON_VERSION=3.9
ARG PYENV_DIR=/pyenv

# -----------------------------------------------------------------------------

FROM ubuntu:${UBUNTU_VERSION} AS base-builder

ARG PYENV_DIR

ENV DEBIAN_FRONTEND noninteractive
ENV PYTHONUNBUFFERED 1

RUN set -ex \
&& apt-get update \
&& apt-get install -y --no-install-recommends \
ca-certificates \
curl \
git \
gnupg \
libldap2-dev \
libmysqlclient-dev \
libsasl2-dev \
libsqlite3-dev \
locales \
pkg-config \
tzdata \
&& rm -rf /var/lib/apt/lists/* /var/cache/apt/*

RUN locale-gen en_US.UTF-8
ENV LANG en_US.UTF-8
ENV LANGUAGE en_US:en
ENV LC_ALL en_US.UTF-8

ENV PYENV_ROOT=${PYENV_DIR}/data
ENV PATH=$PYENV_ROOT/shims:$PYENV_ROOT/bin:$PATH

# -----------------------------------------------------------------------------

FROM base-builder AS pyenv-builder

ARG PYTHON_VERSION

RUN set -ex \
&& apt-get update \
&& apt-get install -y --no-install-recommends \
build-essential \
libbz2-dev \
libffi-dev \
liblzma-dev \
libncursesw5-dev \
libreadline-dev \
libsqlite3-dev \
libssl-dev \
libxml2-dev \
libxmlsec1-dev \
tk-dev \
xz-utils \
zlib1g-dev \
&& rm -rf /var/lib/apt/lists/* /var/cache/apt/*

RUN set -ex \
&& curl --retry 3 -L https://github.com/pyenv/pyenv-installer/raw/master/bin/pyenv-installer | bash \
&& pyenv install ${PYTHON_VERSION} \
&& pyenv global ${PYTHON_VERSION}

COPY --link requirements-dev.txt /src/requirements-dev.txt

RUN set -ex \
&& pyenv exec python3 -m pip install --upgrade pip setuptools \
&& pyenv exec python3 -m pip install --requirement /src/requirements-dev.txt \
&& pyenv rehash


# -----------------------------------------------------------------------------

FROM base-builder as base

ARG USER_ID
ARG GROUP_ID
ARG PYENV_DIR

RUN set -ex \
&& curl --retry 3 -fsSL https://packages.archivematica.org/1.16.x/key.asc | gpg --dearmor -o /etc/apt/keyrings/archivematica-1.16.x.gpg \
&& echo "deb [arch=amd64 signed-by=/etc/apt/keyrings/archivematica-1.16.x.gpg] http://packages.archivematica.org/1.16.x/ubuntu-externals jammy main" > /etc/apt/sources.list.d/archivematica-external.list \
&& curl --retry 3 -so /tmp/repo-mediaarea_1.0-21_all.deb -L https://mediaarea.net/repo/deb/repo-mediaarea_1.0-21_all.deb \
&& dpkg -i /tmp/repo-mediaarea_1.0-21_all.deb \
&& apt-get update \
&& apt-get install -y --no-install-recommends \
atool \
bulk-extractor \
clamav \
coreutils \
ffmpeg \
fits \
g++ \
gcc \
gearman \
gettext \
ghostscript \
hashdeep \
imagemagick \
inkscape \
jhove \
libffi-dev \
libimage-exiftool-perl \
libldap2-dev \
libmysqlclient-dev \
libsasl2-dev \
libssl-dev \
libxml2-dev \
libxslt1-dev \
logapp \
md5deep \
mediaconch \
mediainfo \
nailgun \
nfs-common \
openjdk-8-jre-headless \
p7zip-full \
pbzip2 \
pst-utils \
python3-lxml \
rsync \
siegfried \
sleuthkit \
tesseract-ocr \
tree \
unar \
unrar-free \
uuid \
&& rm -rf /var/lib/apt/lists/* /var/cache/apt/*

RUN set -ex \
&& groupadd --gid ${GROUP_ID} --system archivematica \
&& useradd --uid ${USER_ID} --gid ${GROUP_ID} --home-dir /var/archivematica --system archivematica \
&& mkdir -p /var/archivematica/sharedDirectory \
&& chown -R archivematica:archivematica /var/archivematica

# Download ClamAV virus signatures
RUN freshclam --quiet

USER archivematica

COPY --chown=${USER_ID}:${GROUP_ID} --from=pyenv-builder --link ${PYENV_DIR} ${PYENV_DIR}
COPY --chown=${USER_ID}:${GROUP_ID} --link . /src

# -----------------------------------------------------------------------------

FROM base

ENV DJANGO_SETTINGS_MODULE settings.common
ENV PYTHONPATH /src/src/MCPClient/lib/:/src/src/MCPClient/lib/clientScripts:/src/src/archivematicaCommon/lib/:/src/src/dashboard/src/
ENV ARCHIVEMATICA_MCPCLIENT_ARCHIVEMATICACLIENTMODULES /src/src/MCPClient/lib/archivematicaClientModules
ENV ARCHIVEMATICA_MCPCLIENT_CLIENTASSETSDIRECTORY /src/src/MCPClient/lib/assets/
ENV ARCHIVEMATICA_MCPCLIENT_CLIENTSCRIPTSDIRECTORY /src/src/MCPClient/lib/clientScripts/

# Some scripts in archivematica-fpr-admin executed by MCPClient rely on certain
# files being available in this image (e.g. see https://git.io/vA1wF).
COPY --link src/archivematicaCommon/lib/externals/fido/ /usr/lib/archivematica/archivematicaCommon/externals/fido/
COPY --link src/archivematicaCommon/lib/externals/fiwalk_plugins/ /usr/lib/archivematica/archivematicaCommon/externals/fiwalk_plugins/

ENTRYPOINT ["pyenv", "exec", "python3", "/src/src/MCPClient/lib/archivematicaClient.py"]
File renamed without changes.
11 changes: 0 additions & 11 deletions hack/ccp/integration/main_test.go

This file was deleted.

57 changes: 0 additions & 57 deletions hack/ccp/integration/mysql_test.go

This file was deleted.

Loading

0 comments on commit 271a9a4

Please sign in to comment.