Skip to content

Commit

Permalink
Merge pull request #3034 from OCamlPro/release-building
Browse files Browse the repository at this point in the history
Add rules for automatised release builds
  • Loading branch information
AltGr committed Aug 23, 2017
2 parents 4e41196 + 26c2454 commit 84b97b6
Show file tree
Hide file tree
Showing 9 changed files with 172 additions and 58 deletions.
3 changes: 1 addition & 2 deletions .travis-ci.sh
Original file line number Diff line number Diff line change
Expand Up @@ -13,10 +13,9 @@ install-bootstrap () {
eval $(opam config env --root=$OPAMBSROOT)
if [ "$OPAM_TEST" = "1" ]; then
# TEMPORARY: waiting for merge into opam-repository
opam pin add jbuilder.1.0+beta12 "https://github.com/janestreet/jbuilder/archive/b913a42739362ac58c9a3df55a80eeacc9af9135.tar.gz" --yes --no-action
( cd $(OPAMBSROOT) && git clone https://github.com/AltGr/ocaml-mccs && cd ocaml-mccs && git checkout 1.1+2b && opam pin add mccs.1.1+2 . --yes; )

opam install ocamlfind ocamlbuild cohttp cohttp-lwt-unix ssl cmdliner dose3 opam-file-format re jbuilder mccs --yes
opam install ocamlfind ocamlbuild cohttp cohttp-lwt-unix ssl cmdliner dose3 opam-file-format re jbuilder.1.0-beta12 mccs --yes
# Allow use of ocamlfind packages in ~/local/lib
FINDCONF=$(ocamlfind printconf conf)
sed "s%^path=.*%path=\"$HOME/local/lib:$(opam config var lib)\"%" $FINDCONF >$FINDCONF.1
Expand Down
37 changes: 16 additions & 21 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -28,27 +28,24 @@ src_ext/jbuilder.stamp:
$(MAKE) -C src_ext jbuilder.stamp

jbuilder: $(JBUILDER_DEP)
@$(JBUILDER) build @install
@$(JBUILDER) build $(JBUILDER_ARGS) @install

opam: $(JBUILDER_DEP) opam.install
ln -sf _build/default/src/client/opamMain.exe $@

%-static: ALWAYS
OPAM_BUILD_STATIC=true $(MAKE) $*

opam-installer: $(JBUILDER_DEP)
$(JBUILDER) build src/tools/opam_installer.exe
$(JBUILDER) build $(JBUILDER_ARGS) src/tools/opam_installer.exe
ln -sf _build/default/src/tools/opam_installer.exe $@

opam-admin.top: $(JBUILDER_DEP)
$(JBUILDER) build src/tools/opam_admin_top.bc
$(JBUILDER) build $(JBUILDER_ARGS) src/tools/opam_admin_top.bc
ln -sf _build/default/src/tools/opam_admin_top.bc $@

lib-ext:
$(MAKE) -j -C src_ext lib-ext

download-ext:
$(MAKE) -C src_ext archives
$(MAKE) -C src_ext cache-archives

clean-ext:
$(MAKE) -C src_ext distclean
Expand Down Expand Up @@ -81,14 +78,14 @@ ifneq ($(LIBINSTALL_DIR),)
endif

opam-devel.install: $(JBUILDER_DEP)
$(JBUILDER) build -p opam opam.install
$(JBUILDER) build $(JBUILDER_ARGS) -p opam opam.install
sed -e "s/bin:/libexec:/" opam.install > $@

opam-%.install: $(JBUILDER_DEP)
$(JBUILDER) build -p opam-$* $@
$(JBUILDER) build $(JBUILDER_ARGS) -p opam-$* $@

opam.install: $(JBUILDER_DEP)
$(JBUILDER) build $@
$(JBUILDER) build $(JBUILDER_ARGS) $@

opam-actual.install: opam.install
@sed -n -e "/^bin: /,/^]/p" $< > $@
Expand All @@ -105,35 +102,35 @@ opam-actual.install: opam.install
OPAMLIBS = core format solver repository state client

opam-%: $(JBUILDER_DEP)
$(JBUILDER) build opam-$*.install
$(JBUILDER) build $(JBUILDER_ARGS) opam-$*.install

opam-lib: $(JBUILDER_DEP)
$(JBUILDER) build $(patsubst %,opam-%.install,$(OPAMLIBS))
$(JBUILDER) build $(JBUILDER_ARGS) $(patsubst %,opam-%.install,$(OPAMLIBS))

installlib-%: $(JBUILDER_DEP) opam-installer opam-%.install
$(if $(wildcard src_ext/lib/*),\
$(error Installing the opam libraries is incompatible with embedding \
the dependencies. Run 'make clean-ext' and try again))
$(JBUILDER) exec -- opam-installer $(OPAMINSTALLER_FLAGS) opam-$*.install
$(JBUILDER) exec $(JBUILDER_ARGS) -- opam-installer $(OPAMINSTALLER_FLAGS) opam-$*.install

uninstalllib-%: $(JBUILDER_DEP) opam-installer opam-%.install
$(JBUILDER) exec -- opam-installer -u $(OPAMINSTALLER_FLAGS) opam-$*.install
$(JBUILDER) exec $(JBUILDER_ARGS) -- opam-installer -u $(OPAMINSTALLER_FLAGS) opam-$*.install

libinstall: $(JBUILDER_DEP) opam-admin.top $(OPAMLIBS:%=installlib-%)
@

install: opam-actual.install $(JBUILDER_DEP)
$(JBUILDER) exec -- opam-installer $(OPAMINSTALLER_FLAGS) $<
$(JBUILDER) exec $(JBUILDER_ARGS) -- opam-installer $(OPAMINSTALLER_FLAGS) $<

libuninstall: $(OPAMLIBS:%=uninstalllib-%)
@

uninstall: opam-actual.install $(JBUILDER_DEP)
$(JBUILDER) exec -- opam-installer -u $(OPAMINSTALLER_FLAGS) $<
$(JBUILDER) exec $(JBUILDER_ARGS) -- opam-installer -u $(OPAMINSTALLER_FLAGS) $<

.PHONY: tests tests-local tests-git
tests: $(JBUILDER_DEP)
$(JBUILDER) runtest
$(JBUILDER) runtest $(JBUILDER_ARGS)

# tests-local, tests-git
tests-%:
Expand All @@ -151,10 +148,8 @@ configure: configure.ac m4/*.m4
aclocal -I m4
autoconf

release-tag:
git tag -d latest || true
git tag -a latest -m "Latest release"
git tag -a $(version) -m "Release $(version)"
release-%:
$(MAKE) -C release TAG="$*"

cold:
env MAKE=$(MAKE) ./shell/bootstrap-ocaml.sh
Expand Down
29 changes: 29 additions & 0 deletions release/Dockerfile.in
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
FROM multiarch/debian-debootstrap:%TARGET_TAG%
# May need configuration on the host:
# docker run --rm --privileged multiarch/qemu-user-static:register --reset
LABEL Description="opam release builds" Vendor="OCamlPro" Version="1.0"

RUN apt-get update && apt-get install bzip2 g++ make patch wget libglpk-dev libltdl-dev --yes && apt-get clean --yes
RUN useradd -U --create-home opam

ADD https://caml.inria.fr/pub/distrib/ocaml-4.04/ocaml-4.04.2.tar.gz /root/

WORKDIR /root
RUN tar xzf ocaml-4.04.2.tar.gz
WORKDIR ocaml-4.04.2
RUN ./configure %CONF% -prefix /usr/local
RUN make world opt.opt
RUN make install
RUN rm -rf /root/ocaml-4.04.2 /root/ocaml-4.04.2.tar.gz

ENV PATH /usr/local/bin:/usr/bin:/bin
USER opam
VOLUME /src
WORKDIR /home/opam/
CMD tar xzf /src/opam-full-${VERSION}.tar.gz && \
cd opam-full-${VERSION} && \
echo "(${LINKING})" > src/client/linking.sexp && \
./configure --with-mccs && \
make lib-ext opam && \
strip opam && \
cp opam /src/opam-${VERSION}-${TARGET}
119 changes: 119 additions & 0 deletions release/Makefile
Original file line number Diff line number Diff line change
@@ -0,0 +1,119 @@
TAG = master
VERSION = $(shell git describe $(TAG) | sed 's/-/~/')
GIT_URL = ..

FULL_ARCHIVE_URL = https://github.com/ocaml/opam/releases/download/$(VERSION)/opam-full-$(VERSION).tar.gz

TARGETS = x86_64-linux i686-linux armhf-linux arm64-linux
# todo: x86_64-darwin

OCAMLV = 4.04.2
# currently hardcoded in Dockerfile.in
OCAML_URL = https://caml.inria.fr/pub/distrib/ocaml-$(basename $(OCAMLV))/ocaml-$(OCAMLV).tar.gz

HOST_OS = $(shell uname -s | tr A-Z a-z)
HOST = $(shell uname -m)-$(HOST_OS)

all: $(patsubst %,out/opam-$(VERSION)-%,$(TARGETS))

out/opam-full-$(VERSION).tar.gz:
mkdir -p out
cd out && curl -OL $(FULL_ARCHIVE_URL) || { \
git clone $(GIT_URL) -b $(TAG) --depth 1 opam-full-$(VERSION); \
sed -i 's/^AC_INIT(opam,.*)/AC_INIT(opam,$(VERSION))/' \
opam-full-$(VERSION)/configure.ac; \
cd opam-full-$(VERSION) && $(MAKE) configure download-ext; \
tar cz --exclude-vcs opam-full-$(VERSION) -f $@; \
rm -rf opam-full-$(VERSION); \
}

build/Dockerfile.x86_64-linux: Dockerfile.in
mkdir -p build && sed 's/%TARGET_TAG%/amd64-jessie/g' $^ | sed 's/%CONF%//g' >$@
build/Dockerfile.i686-linux: Dockerfile.in
mkdir -p build && sed 's/%TARGET_TAG%/i386-jessie/g' $^ | sed 's/%CONF%/-host i686-linux/g' >$@
build/Dockerfile.armhf-linux: Dockerfile.in
mkdir -p build && sed 's/%TARGET_TAG%/armhf-jessie/g' $^ | sed 's/%CONF%//g' >$@
build/Dockerfile.arm64-linux: Dockerfile.in
mkdir -p build && sed 's/%TARGET_TAG%/arm64-jessie/g' $^ | sed 's/%CONF%//g' >$@


build/%.image: build/Dockerfile.%
docker build -t opam-build-$* -f $^ build
touch $@

# Actually, this is for debian 8 jessie, and varies wildly
CLINKING_linux = \
-Wl,-Bstatic \
-lunix -lbigarray -lmccs_stubs \
-lglpk -lstdc++ -lz -lamd -lcolamd -lltdl \
-Wl,-Bdynamic \
-static-libgcc

CLINKING_darwin = \
-lunix -lbigarray -lmccs_stubs \
/usr/local/lib/libglpk.a -lz -lstdc++

CLINKING_openbsd = $(CLINKING_darwin)

EXPORTS_openbsd = \
CPATH=/usr/local/include: \
LIBRARY_PATH=/usr/local/lib: \


%: opam-$(VERSION)-%

opam-$(VERSION)-%: out/opam-$(VERSION)-%
ln -sf $^ $@

host: opam-$(VERSION)-$(HOST)

# Build for the local host. Containerised builds, below, are preferred, but not always available
build/$(HOST).env:
mkdir -p build/$(HOST)
cd build/$(HOST) && curl -OL $(OCAML_URL)
cd build/$(HOST) && tar xzf ocaml-$(OCAMLV).tar.gz
cd build/$(HOST)/ocaml-$(OCAMLV) && \
./configure -prefix $(shell pwd)/build/$(HOST) && \
$(MAKE) world opt.opt && \
$(MAKE) install
rm -rf build/$(HOST)/ocaml-$(OCAMLV) build/$(HOST)/ocaml-$(OCAMLV).tar.gz
touch $@

out/opam-$(VERSION)-$(HOST): out/opam-full-$(VERSION).tar.gz build/$(HOST).env
cd build && tar xzf ../$<
( export \
PATH=$(shell pwd)/build/$(HOST)/bin:$$PATH \
MAKE=$(MAKE) \
$(EXPORTS_$(HOST_OS)); \
cd build/opam-full-$(VERSION) && \
./configure --with-glpk && \
echo "(-noautolink $(patsubst %,-cclib %,$(CLINKING_$(HOST_OS))))" \
>src/client/linking.sexp && \
$(MAKE) lib-ext opam JBUILDER_ARGS="--root=`pwd`"; \
)
strip build/opam-full-$(VERSION)/opam
cp build/opam-full-$(VERSION)/opam out/opam-$(VERSION)-$(HOST)
rm -rf build/opam-full-$(VERSION)

# Containerised builds
out/opam-$(VERSION)-%-linux: build/%-linux.image out/opam-full-$(VERSION).tar.gz
docker run --rm -v `pwd`/out:/src \
-e "VERSION=$(VERSION)" \
-e "TARGET=$*-linux" \
-e "LINKING=$(LINKING_jessie)" \
opam-build-$*-linux

clean:
rm -rf build

distclean: clean
rm -rf out

REMOTE_DIR = /tmp/opam-release
REMOTE_MAKE = gmake
remote: out/opam-full-$(VERSION).tar.gz
ssh "$(REMOTE)" "mkdir -p $(REMOTE_DIR)/out"
scp Makefile "$(REMOTE):$(REMOTE_DIR)/"
scp "$^" "$(REMOTE):$(REMOTE_DIR)/$^"
ssh "$(REMOTE)" 'sh -c "cd $(REMOTE_DIR) && $(REMOTE_MAKE) host TAG=$(TAG) VERSION=$(VERSION) OCAMLV=$(OCAMLV)"'
scp "$(REMOTE):$(REMOTE_DIR)/out/opam-$(VERSION)*" out/
19 changes: 4 additions & 15 deletions src/client/jbuild
Original file line number Diff line number Diff line change
Expand Up @@ -28,16 +28,6 @@ let (sha, version) =
remove ();
("", "let version = None")

let static = try match Sys.getenv "OPAM_BUILD_STATIC" with
| "1" | "yes" | "true" ->
(try match List.assoc "system" J.ocamlc_config with
| "macosx" -> "clang"
| _ -> "gcc"
with Not_found -> failwith "Could not detect platform for static linking")
| "" | "0" | "no" | "false" -> "false"
| _ -> failwith "OPAM_BUILD_STATIC: invalid value"
with Not_found -> "false"

let () = Printf.ksprintf J.send {|
(jbuild_version 1)

Expand All @@ -60,7 +50,7 @@ let () = Printf.ksprintf J.send {|
(flags (:standard
(:include ../ocaml-flags-standard.sexp)
(:include ../ocaml-context-flags.sexp)
(:include static-linking.sexp)))
(:include linking.sexp)))
(libraries (opam-client))))

(rule
Expand All @@ -70,7 +60,6 @@ let () = Printf.ksprintf J.send {|
(with-stdout-to git-sha-%s (echo "%s")))

(rule
((targets (static-linking.sexp))
(deps (static-linking.sexp.%s))
(action (run cp ${<} ${@}))))
|} sha sha version static
((targets (linking.sexp))
(action (with-stdout-to ${@} (run echo "()")))))
|} sha sha version
7 changes: 0 additions & 7 deletions src/client/static-linking.sexp.clang

This file was deleted.

1 change: 0 additions & 1 deletion src/client/static-linking.sexp.false

This file was deleted.

9 changes: 0 additions & 9 deletions src/client/static-linking.sexp.gcc

This file was deleted.

6 changes: 3 additions & 3 deletions src_ext/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ ifneq ($(filter-out archives cache-archives,$(MAKECMDGOALS)),)
-include ../Makefile.config
endif

SRC_EXTS = cppo extlib re cmdliner ocamlgraph cudf dose3 opam-file-format result
SRC_EXTS = cppo extlib re cmdliner ocamlgraph cudf dose3 opam-file-format result jbuilder
ifneq ($(MCCS_DISABLED),true)
SRC_EXTS := $(SRC_EXTS) mccs
endif
Expand Down Expand Up @@ -37,8 +37,8 @@ MD5_opam-file-format = fb461d14a44aac3a43751aa936e79143
URL_result = https://github.com/janestreet/result/archive/1.2.tar.gz
MD5_result = 3d5b66c5526918f0f2ca9d6811ef09c8

URL_jbuilder = https://github.com/dra27/jbuilder/archive/more-reads.tar.gz
MD5_jbuilder = 6c329177bd81a0b25ebde7235eeb19bc
URL_jbuilder = https://github.com/janestreet/jbuilder/archive/1.0+beta12.tar.gz
MD5_jbuilder = 05dcb6bdba0c99ae30b63237170e474a

ifndef FETCH
ifneq ($(shell command -v curl 2>/dev/null),)
Expand Down

0 comments on commit 84b97b6

Please sign in to comment.