diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..19d7311 --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +docs/_build +docs/_static +docs/_templates +utils/ diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..cd154f8 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,26 @@ +FROM centos:centos7 + +# +# Install 'fpm' dependencies +# +RUN yum install -q -y ruby-devel gcc make rpm-build rubygems + +# +# Install 'fpm' +# +RUN gem install --no-ri --no-rdoc fpm + +# +# Create non-privileged user +# +ENV username="lsstsw" +RUN useradd --create-home --uid 1000 --user-group --home-dir /home/${username} ${username} + +# +# Add files needed to build the packages +# +WORKDIR /home/${username} +ADD ["buildLinuxPkg.sh", "version.sh", "./"] +ADD ["etc/", "./etc/"] + +ENTRYPOINT /home/${username}/buildLinuxPkg.sh diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..3f7d5f9 --- /dev/null +++ b/LICENSE @@ -0,0 +1,13 @@ +Copyright 2018 Fabio Hernandez + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..c976958 --- /dev/null +++ b/Makefile @@ -0,0 +1,25 @@ +UNAME := $(shell uname) + +.PHONY: all + +all: + @echo "Usage: make " + + +pkg: +ifeq ($(UNAME), Linux) + docker build -t airnandez/sw-lsst-eu . + docker run -it --mount type=bind,source=`pwd`,destination=/scratch --env SCRATCH=/scratch airnandez/sw-lsst-eu +else + bash buildMacOSPkg.sh +endif + + +test: +ifeq ($(UNAME), Linux) + docker run -it --entrypoint /bin/bash airnandez/sw-lsst-eu +endif + + +clean: + @rm -f ./*.deb ./*.rpm diff --git a/README.md b/README.md index 7cfd6c0..ac8bd3b 100644 --- a/README.md +++ b/README.md @@ -1,111 +1,22 @@ -LSST FRANCE DOCUMENTATION -========================= +# Cloud-based delivery of the LSST science pipelines software -
+## Overview +This repository contains documentation and tools on how to get the [Large Synoptic Survey Telescope (LSST) science pipelines](https://pipelines.lsst.io) software delivered to your personal computer on demand. -Introduction ----------- -*** +All the details available at [https://sw.lsst.eu](https://sw.lsst.eu). -This repository contains LSST documentation sources (reStructuredText file format). -Updating content on this repository automatically generates and publishes online documentation. +## License -
+Copyright 2018 Fabio Hernandez -Goals ----------- -*** +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at -- Manage collaboration between LSST documention's authors -- Generate & Publish LSST Documentation +[http://www.apache.org/licenses/LICENSE-2.0](http://www.apache.org/licenses/LICENSE-2.0) -
- -Workflow ----------- -*** - -Each author works on his own branch and can see a preview at every push. -Once the author is satisfied, he can merge his branch with the master branch. -The merge in master generates and publishes the official version. -This repository only contains source documentation (.rst files). -Authors must write documentation respecting this format. -More info on the reStructuredText (rst) file format: https://en.wikipedia.org/wiki/ReStructuredText - -![LSST France Software Doc Diagram](lsst-france-sw-diagram.png "LSST France Software Doc Diagram") - -### master branch is the official doc that is pushed online at http://sw.lsst.eu/. -### Documentation generation and uploading is fully automated. - -
- -Instructions ----------- -*** - -**Step1**: clone the repo and create your own branch from master - -```bash -git clone git@gitlab.in2p3.fr:brigaud/sw-lsst-eu.git -cd sw-lsst-eu -git checkout -b -``` - -**Step2**: create/modify your doc in your branch then push it - -```bash -git add --all -git commit -m "your changes comments" -git push -``` -... wait about a minute and check it out at http://sw.lsst.eu/preview/\. - -
- -> **Note:** - -> - Each push on your branch will generate a documentation at http://sw.lsst.eu/preview/\ -> - Useful for preview. - -
- -**Step3**: send a merge request to master branch to put documentation in production -

-https://gitlab.in2p3.fr/brigaud/sw-lsst-eu/merge_requests/new?merge_request%5Bsource_branch%5D=\ -

-An administrator will check your version, fix conflicts and finally merge your branch into master branch.
-This merge will automatically trigger the build process and update the official documentation at http://sw.lsst.eu/. - -
- -> **Note:** - -> - Each merge on master branch will generate a documentation at http://sw.lsst.eu - -
- -> **Important**: - -> - Each time master builds a new official doc, preview folder (i.e http://sw.lsst.eu/preview) is deleted - -
- -Recommandations --------------------- -*** - -All the documentation resides in the "docs" folder. -You can create or delete files and folders in order to create your documentation. -Be sure to update the index.rst file in case you add or delete documentation files or folders in order to access it from the sidebar menu on the website. - -
- -> **Note:** - -> - As a community project, it is recommended to work only on parts you've been assigned -> - Keep as much as possible the original tree structure -> - Keep as much as possible your branch up-to-date with respect to master branch (i.e git rebase master ) - -
- -Happy doc! +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/buildLinuxPkg.sh b/buildLinuxPkg.sh new file mode 100755 index 0000000..9969444 --- /dev/null +++ b/buildLinuxPkg.sh @@ -0,0 +1,51 @@ +#!/bin/bash + +# +# Import version +# +source 'version.sh' +version=$(pkgVersion) + +# +# Package name +# +packageName="cvmfs-config-lsst" + +# +# Prepare the output directory +# +outputDir=${SCRATCH:-/tmp} +mkdir -p ${outputDir} + +# +# Clean up first +# +rm -f ${outputDir}/${packageName}*.{rpm,deb} + +# +# Build RPM and DEB packages +# +for out in rpm deb; do + fpm --name ${packageName} \ + --version ${version} \ + --depends cvmfs \ + --description "Configuration files for CernVM-FS file system /cvmfs/sw.lsst.eu" \ + --maintainer "Fabio Hernandez (fabio@in2p3.fr)" \ + --url "https://sw.lsst.eu" \ + --license "Apache v2.0" \ + --vendor "CNRS / IN2P3 computing center (CC-IN2P3)" \ + --architecture all \ + --output-type ${out} \ + --input-type dir \ + --package ${outputDir} \ + --config-files ./etc \ + ./etc + done + +# +# Set file ownership, if necessary +# +userName="lsstsw" +if getent passwd ${userName} > /dev/null 2>&1; then + chown ${userName}:${userName} ${outputDir}/${packageName}*.{rpm,deb} +fi diff --git a/buildMacOSPkg.sh b/buildMacOSPkg.sh new file mode 100755 index 0000000..06bdf8d --- /dev/null +++ b/buildMacOSPkg.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +# +# Import version +# +source 'version.sh' +version=$(pkgVersion) + +# +# Package id +# +packageId="eu.lsst.sw.cvmfs.config" + +# +# Package file name +# +packageFileName="sw-lsst-eu-cvmfs-config_${version}.pkg" + +# +# Build the package +# +sudo pkgbuild \ + --root ./etc \ + --identifier ${packageId} \ + --version ${version} \ + --install-location /etc \ + ${packageFileName} \ No newline at end of file diff --git a/check.sh b/check.sh new file mode 100644 index 0000000..54c1b8e --- /dev/null +++ b/check.sh @@ -0,0 +1,37 @@ +#!/bin/bash + +# +# Usage: bash check.sh +# +# Purpose: test the reachability from this host of the configured HTTP servers +# for CernVM-FS software repository 'sw.lsst.eu', via the configured proxies +# + +if [[ ! -f /etc/cvmfs/domain.d/lsst.eu.conf ]]; then + echo "file /etc/cvmfs/domain.d/lsst.eu.conf could not be found" + exit 1 +fi + +curlCmd=$(command -v curl) +if [[ -z ${curlCmd} ]]; then + echo "command curl not found" + exit 1 +fi + +source /etc/cvmfs/domain.d/lsst.eu.conf + +urls=`echo ${CVMFS_SERVER_URL} | sed -e 's/;/ /g' -e 's|@fqrn@|sw.lsst.eu/.cvmfspublished|g'` +proxies=`echo ${CVMFS_HTTP_PROXY} | sed -e 's/|/ /g' -e 's/;/ /g'` + +rc=0 +for u in ${urls}; do + for p in ${proxies}; do + ok=`${curlCmd} --silent --head --proxy ${p} ${u} | grep "HTTP/1.1 200 OK"` + if [[ -z ${ok} ]]; then + echo "HTTP HEAD request for $u via proxy $p failed" + rc=1 + fi + done +done + +exit $rc \ No newline at end of file diff --git a/configure.sh b/configure.sh new file mode 100644 index 0000000..c663e4e --- /dev/null +++ b/configure.sh @@ -0,0 +1,66 @@ +#!/bin/bash + +# +# Usage: sudo bash configure.sh +# +# This script configures a CernVM FS client to access the +# LSST software repository /cvmfs/sw.lsst.eu. The CernVM-FS +# client software must be previously installed. +# + +usage() { + echo "Usage: sudo bash configure.sh" +} + +# +# We must run as 'root' +# +if [[ $EUID -ne 0 ]]; then + usage + exit 1 +fi + +# +# Execute 'cvmfs_config setup' +# +cvmfscfg=$(command -v cvmfs_config) +if [[ -z ${cvmfscfg} ]] || [[ ! -x ${cvmfscfg} ]]; then + echo "Could not find CernVM FS configuration command 'cvmfs_config'" + exit 1 +fi +${cvmfscfg} setup +if [ $? -ne 0 ]; then + exit 1 +fi + +# +# Copy configuration files to their destination under +# /etc/cvmfs +# +srcDir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +destDir='/' +toInstall="etc/cvmfs/domain.d etc/cvmfs/keys/lsst.eu" +for dir in ${toInstall}; do + dest=$(readlink -m ${destDir}/${dir}) + install -d ${dest} + install --backup --compare --mode=u=r,g=r,o=r -D ${srcDir}/${dir}/* ${dest} +done + +# +# Perform system-specific tasks +# +thisOS=`uname` +if [ "$thisOS" == "Linux" ]; then + # Use 'cvmfs_config' to check the configuration + result=`${cvmfscfg} chksetup` + if [ "$result" != "OK" ]; then + echo "There was an error checking your CernVM FS configuration:" + echo $result + exit 1 + fi +elif [ "$thisOS" == "Darwin" ]; then + # On MacOS X, create the mount directory + mkdir -p /cvmfs/sw.lsst.eu +fi + +exit 0 diff --git a/devdocs/README.md b/devdocs/README.md new file mode 100644 index 0000000..70ddbb1 --- /dev/null +++ b/devdocs/README.md @@ -0,0 +1,30 @@ +# Developer documentation + +This document contain instructions on how to trigger the continuous integration machinery for building the documentation of +site [https://sw.lsst.eu](https://sw.lsst.eu). + +The sources of the documentation are located in the `docs` directory of this repository. To build them do: + +```bash +git clone git@gitlab.in2p3.fr:brigaud/sw-lsst-eu.git +cd sw-lsst-eu +git checkout -b +``` + +The last step, i.e. creating your own branch is very important. You can now modify the contents of your local copy of the files in the `docs` directory and commit your changes. Then do: + +```bash +git push +``` + +This push operation will trigger the CI infrastructure. You can preview the modified version of the documentation at the address: `https://sw.lsst.eu/preview/`. + +Once you are happy with your modifications and want to push them in production make a merge request via the command line or by visiting: + +``` +https://gitlab.in2p3.fr/brigaud/sw-lsst-eu/merge_requests/new?merge_request%5Bsource_branch%5D= +``` + +You can have an overview of the process in the diagram below: + +![Workflow](workflow.png "Workflow") \ No newline at end of file diff --git a/lsst-france-sw-diagram.png b/devdocs/workflow.png similarity index 100% rename from lsst-france-sw-diagram.png rename to devdocs/workflow.png diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 0000000..64d14ff --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = +SPHINXBUILD = sphinx-build +SPHINXPROJ = swlssteu +SOURCEDIR = . +BUILDDIR = _build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) \ No newline at end of file diff --git a/docs/conf.py b/docs/conf.py new file mode 100644 index 0000000..a9b8617 --- /dev/null +++ b/docs/conf.py @@ -0,0 +1,171 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# sw.lsst.eu documentation build configuration file, created by +# sphinx-quickstart on Wed Apr 18 17:55:28 2018. +# +# This file is execfile()d with the current directory set to its +# containing dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +# import os +# import sys +# sys.path.insert(0, os.path.abspath('.')) + + +# -- General configuration ------------------------------------------------ + +# If your documentation needs a minimal Sphinx version, state it here. +# +# needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = ['sphinx.ext.todo', + 'sphinx.ext.coverage'] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# The suffix(es) of source filenames. +# You can specify multiple suffix as a list of string: +# +# source_suffix = ['.rst', '.md'] +source_suffix = '.rst' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +project = 'sw.lsst.eu' +copyright = '2018, Fabio Hernandez' +author = 'Fabio Hernandez' + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. +# +# The short X.Y version. +version = '0.1' +# The full version, including alpha/beta/rc tags. +release = '0.1' + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language = None + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This patterns also effect to html_static_path and html_extra_path +exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + +# If true, `todo` and `todoList` produce output, else they produce nothing. +todo_include_todos = True + + +# -- Options for HTML output ---------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +# html_theme = 'alabaster' +html_theme = "sphinx_rtd_theme" + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +# +# html_theme_options = {} + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] + +# Custom sidebar templates, must be a dictionary that maps document names +# to template names. +# +# This is required for the alabaster theme +# refs: http://alabaster.readthedocs.io/en/latest/installation.html#sidebars +html_sidebars = { + '**': [ + 'relations.html', # needs 'show_related': True theme option to display + 'searchbox.html', + ] +} + + +# -- Options for HTMLHelp output ------------------------------------------ + +# Output file base name for HTML help builder. +htmlhelp_basename = 'swlssteudoc' + + +# -- Options for LaTeX output --------------------------------------------- + +latex_elements = { + # The paper size ('letterpaper' or 'a4paper'). + # + # 'papersize': 'letterpaper', + + # The font size ('10pt', '11pt' or '12pt'). + # + # 'pointsize': '10pt', + + # Additional stuff for the LaTeX preamble. + # + # 'preamble': '', + + # Latex figure (float) alignment + # + # 'figure_align': 'htbp', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, +# author, documentclass [howto, manual, or own class]). +latex_documents = [ + (master_doc, 'swlssteu.tex', 'sw.lsst.eu Documentation', + 'Fabio Hernandez', 'manual'), +] + + +# -- Options for manual page output --------------------------------------- + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [ + (master_doc, 'swlssteu', 'sw.lsst.eu Documentation', + [author], 1) +] + + +# -- Options for Texinfo output ------------------------------------------- + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + (master_doc, 'swlssteu', 'sw.lsst.eu Documentation', + author, 'swlssteu', 'One line description of project.', + 'Miscellaneous'), +] + + + diff --git a/docs/credits.rst b/docs/credits.rst new file mode 100644 index 0000000..470bdd9 --- /dev/null +++ b/docs/credits.rst @@ -0,0 +1,11 @@ +.. _credits: + +******* +Credits +******* + +This work is a contribution by `LSST-France `_ to the worldwide `LSST community `_. The web server infrastructure is hosted and operated in Lyon (France) by `CNRS / IN2P3 computing center `_ (CC-IN2P3). + +The continous delivery infrastructure of this web site was set up and is maintained by **Bertrand Rigaud**. **Vanessa Hamar** manages CC-IN2P3's CernVM-FS servers infrastructure and **Fabio Hernandez** takes care of building the LSST software, packaging it for distribution and continously improving this documentation. + +For bringing this service to you we leverage the work done by the `LSST Data Management `_ and CERN's `CernVM-FS `_ teams. diff --git a/docs/faq.rst b/docs/faq.rst new file mode 100644 index 0000000..4bb7bd9 --- /dev/null +++ b/docs/faq.rst @@ -0,0 +1,23 @@ +.. _faq: + +************************** +Frequently Asked Questions +************************** + + +How many releases of the LSST software do you plan to keep available online? +---------------------------------------------------------------------------- + +Our current policy is to make available **all the stable releases** and at least the **latest 12 weekly releases** officially issued by the LSST project. Please be aware that **older releases will be removed without notice**. You are therefore encouraged to use a recent weekly release at any time or to stick to a stable release, if you don't need the latest features of the pipelines. + + +How can I learn more about CernVM-FS? +------------------------------------- + +You can refer to the `CernVM File System `_ project home page and to the official `CernVM-FS documentation `_. + + +Where can I get help on using the LSST science pipelines? +--------------------------------------------------------- + +The best place to get help about the LSST software works is the `LSST Community `_ forum. diff --git a/docs/help.rst b/docs/help.rst new file mode 100644 index 0000000..b5d558f --- /dev/null +++ b/docs/help.rst @@ -0,0 +1,17 @@ +.. _help: + +************************************ +How to Get Help and Provide Feedback +************************************ + + +How to get help +--------------- + +If you have questions or want to share your experience using this service, please feel free to comment via the `LSST Community `_ forum. + + +How to provide feedback +----------------------- + +You are welcome to `open an issue `_ if you experience problems installing the software, using this service or understanding the documentation. Similarly, if you have detected a bug, either in the software or in the documentation, you can contribute by opening an issue. \ No newline at end of file diff --git a/docs/images/favicon.ico b/docs/images/favicon.ico new file mode 100644 index 0000000..3fd93d9 Binary files /dev/null and b/docs/images/favicon.ico differ diff --git a/docs/images/lsst-france-logo.png b/docs/images/lsst-france-logo.png new file mode 100644 index 0000000..df41087 Binary files /dev/null and b/docs/images/lsst-france-logo.png differ diff --git a/docs/index.rst b/docs/index.rst index ce30d13..0189052 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1,14 +1,55 @@ -.. LSST User Guide documentation master file, created by - sphinx-quickstart on Thu Feb 23 16:01:53 2017. +.. sw.lsst.eu documentation master file, created by + sphinx-quickstart on Wed Apr 18 17:55:28 2018. You can adapt this file completely to your liking, but it should at least contain the root `toctree` directive. -###################### -LSST-France User Guide -###################### +**************************************** +LSST software delivered to your computer +**************************************** -Welcome to the LSST-France User Guide. Here you will find supplemental information to the official LSST documentation specifically about the activities of the LSST community in France. +Here you will can information on how to get a binary distribution of the `Large Synoptic Survey Telescope (LSST) `_ `science pipelines `_ to use on your personal computer without installing the software. -.. note:: +As a user of this software distribution service, both **stable and weekly releases** of the LSST software appear as if they were locally installed on your computer. Since **new releases just appear under the local path** ``/cvmfs/sw.lsst.eu`` **without you to take any action**, you can focus on using the software, instead of on the technicalities of installing and updating it regularly. - This space is a permanent work in progress. +Visit the :ref:`overview` to get more details on the benefits and intended audience of this service brought to the `LSST community `_ by `LSST-France `_ and `CNRS / IN2P3 computing center `_ (CC-IN2P3). + +.. toctree:: + :caption: OVERVIEW + :maxdepth: 1 + + overview + + +.. toctree:: + :caption: USAGE + :maxdepth: 1 + + usage + + +.. toctree:: + :caption: INSTALLATION + :maxdepth: 1 + + installation + + +.. toctree:: + :caption: HELP & FEEDBACK + :maxdepth: 1 + + help + + +.. toctree:: + :caption: FAQ + :maxdepth: 1 + + faq + + +.. toctree:: + :caption: CREDITS + :maxdepth: 1 + + credits diff --git a/docs/installation.rst b/docs/installation.rst new file mode 100644 index 0000000..2d1d9bf --- /dev/null +++ b/docs/installation.rst @@ -0,0 +1,206 @@ +.. _installation: + +****************************** +Installation and Configuration +****************************** + + +In this section you will find step-by-step instructions for installing the CernVM-FS client software on your computer and to configure it to access the repository at ``/cvmfs/sw.lsst.eu``. To perform this one-time process **you need super-user privileges**. + +After this process is successfully completed, an unprivileged user ``cvmfs`` is created in your computer and several configuration files are located under ``/etc/cvmfs``. In addition, some executable files are installed (e.g. ``cvmfs_config``, ``cvmfs_fsck``, ``cvmfs_talk``). The location of those executables depends on the operating system: on CentOS and Ubuntu they are located in ``/usr/bin`` and on macOS in ``/usr/local/bin``. + +The installation and configuration steps for each target platform are presented below. + +CentOS +====== + +Step 1: Install the CernVM-FS client software +--------------------------------------------- + +Add `CERN's RPM repository `_ to your computer: + +.. code-block:: bash + + sudo yum install -q -y https://ecsft.cern.ch/dist/cvmfs/cvmfs-release/cvmfs-release-latest.noarch.rpm + +Install the CernVM-FS client: + +.. code-block:: bash + + sudo yum install -q -y cvmfs + +.. hint:: + **Manual installation on CentOS**: If you prefer not to add CERN's package repository to your computer or to choose a specific version of the software, you can directly download the desired version of the relevant package from the `CERN package repository `_ and manually install it using ``yum``. + + +Step 2: Configure the LSST repository +------------------------------------- + +Install the configuration package for repository ``/cvmfs/sw.lsst.eu``: + +.. code-block:: bash + + sudo rpm -i https://github.com/airnandez/sw-lsst-eu/releases/download/v0.5/cvmfs-config-lsst-0.5-1.noarch.rpm + +Complete the CernVM-FS client configuration: + +.. code-block:: bash + + sudo /usr/bin/cvmfs_config setup + + +Step 3: Mount ``/cvmfs/sw.lsst.eu`` +----------------------------------- + +On CentOS, the CernVM-FS client uses ``autofs`` for automatically mounting the file system when required and to unmount it when it is no longer needed. We recommend you configure the ``autofs`` service to start at boot time. Here is one way to do that: + +.. code-block:: bash + + sudo chkconfig autofs on + +You can now proceed to :ref:`testinginstallation`. + +Ubuntu +====== + +Step 1: Install the CernVM-FS client software +--------------------------------------------- + +Add `CERN's APT repository `_ to your computer and install the CernVM-FS client: + +.. code-block:: bash + + sudo apt-get install lsb-release + curl -OL https://ecsft.cern.ch/dist/cvmfs/cvmfs-release/cvmfs-release-latest_all.deb + sudo dpkg -i cvmfs-release-latest_all.deb + sudo apt-get update + +Install the CernVM-FS client: + +.. code-block:: bash + + sudo apt-get --yes install cvmfs + +.. hint:: + **Manual installation on Ubuntu**: If you prefer not to add CERN's package repository to your computer or to choose a specific version of the software, you can directly download the desired version of the relevant package from the `CERN package repository `_ and manually install it using ``dpkg``. + + +Step 2: Configure the LSST repository +------------------------------------- + +Install the configuration package for repository ``/cvmfs/sw.lsst.eu``: + +.. code-block:: bash + + curl -OL https://github.com/airnandez/sw-lsst-eu/releases/download/v0.5/cvmfs-config-lsst_0.5_all.deb + sudo dpkg -i cvmfs-config-lsst_0.5_all.deb + +Complete the CernVM-FS client configuration: + +.. code-block:: bash + + sudo /usr/bin/cvmfs_config setup + + +Step 3: Mount ``/cvmfs/sw.lsst.eu`` +----------------------------------- + +On Ubuntu, the CernVM-FS client uses ``autofs`` for automatically mounting the file system when required and to unmount it when it is no longer needed. We recommend you configure the ``autofs`` service to start at boot time. Here is one way to do that: + +.. code-block:: bash + + sudo sysv-rc-conf autofs on + +You can now proceed to :ref:`testinginstallation`. + +macOS +===== + +Step 1: Install the CernVM-FS client software +--------------------------------------------- + +Download and install the latest stable release of `FUSE for OS X `_. This is a dependency of the CernVM-FS client. + +Install the CernVM-FS client: + +.. code-block:: bash + + curl -OL https://ecsft.cern.ch/dist/cvmfs/cvmfs-2.5.0/cvmfs-2.5.0.pkg + open cvmfs-2.5.0.pkg + + +Step 2: Configure the LSST repository +------------------------------------- + +Install the configuration package for repository ``/cvmfs/sw.lsst.eu``: + +.. code-block:: bash + + curl -OL https://github.com/airnandez/sw-lsst-eu/releases/download/v0.5/sw-lsst-eu-cvmfs-config_0.5.pkg + open sw-lsst-eu-cvmfs-config_0.5.pkg + +Complete the CernVM-FS client configuration: + +.. code-block:: bash + + sudo /usr/bin/cvmfs_config setup + + +Step 3: Mount ``/cvmfs/sw.lsst.eu`` +----------------------------------- + +Create the mount directory: + +.. code-block:: bash + + sudo mkdir -p /cvmfs/sw.lsst.eu + +On macOS you need to manually mount and unmount the file system when needed. To mount the file system do: + +.. code-block:: bash + + sudo mount -t cvmfs sw.lsst.eu /cvmfs/sw.lsst.eu + +and to unmount it: + +.. code-block:: bash + + sudo umount /cvmfs/sw.lsst.eu + + +.. _testinginstallation: + +************************* +Testing your Installation +************************* + +In the previous steps you installed the CernVM-FS client software and configured it to mount the LSST repository. At this point you can check your computer is correctly configured to access ``/cvmfs/sw.lsst.eu`` by doing: + +.. code-block:: bash + + ls /cvmfs/sw.lsst.eu + +If you can see the contents of that directory your computer is correctly configured and you are ready to start using the LSST software. See :ref:`usage` for details. + +.. important:: + + Please note that on both Linux and macOS **you must mount the file system on the directory** ``/cvmfs/sw.lsst.eu`` because the LSST software is specifically built and packaged to be used under this path. The software won't work when relocated under another path. + +.. _troubleshooting: + +********************************* +Troubleshooting your Installation +********************************* + +In order for this distribution mechanism to work, your computer must be connected to the network, be able to establish network connections to the servers operated by CC-IN2P3 and download files via the HTTP protocol. To check that this is the case, please do: + +.. code-block:: bash + + git clone https://github.com/airnandez/sw-lsst-eu + cd sw-lsst-eu + bash check.sh + +You can tell everything is OK if there is no error message. Otherwise, the displayed error message may help you understanding what is wrong. + +If this does not help solving your issue, please see :ref:`help`. + diff --git a/docs/overview.rst b/docs/overview.rst new file mode 100644 index 0000000..95378c3 --- /dev/null +++ b/docs/overview.rst @@ -0,0 +1,55 @@ +.. _overview: + +******** +Overview +******** + +With this cloud-based software distribution mechanism **you will be able to use any of the releases of the LSST science pipelines available online**. Those releases are accessible in **read-only mode through a file system** mounted on path ``/cvmfs/sw.lsst.eu`` and effectively appear as if you had installed all of them on your computer. + +This service is built upon CERN's `CernVM file system `_, a POSIX read-only file system in user space designed for large-scale software distribution. A software agent running on your computer interacts with servers via standard protocols, downloads the files on demand, caches them on your computer and exposes them via the operating system's file system interface. + +In this section you will find information on the benefits of this mechanism, its intended audience, the platforms where it is supported and how to get started. + +======== +Benefits +======== + +This service mainly intends to make easier the use of the LSST software by a broader community: the more people test and routinely use the software the better it will get. + +Since keeping up with the frequent releases of the LSST software by installing it yourself can be time consuming, with this service **you can focus on using the LSST software from the comfort of your personnal computer**, instead on the technicalities of frequently installing and updating it. + +At any given moment you will find several releases, both **stable** and **weekly**, of the LSST software ready to use. You just need to navigate the file system under the path + +.. code-block:: bash + + /cvmfs/sw.lsst.eu + +to select the release you want to use, configure your environment to use the desired release and start using the software. This approach has the additional benefit that you can test your own scripts and notebooks against several releases of the LSST software, compare results, use new features, detect regresions and provide feedback to the developers. + +Each one of releases you find under ``/cvmfs/sw.lsst.eu`` is packaged so that it is almost self-contained. In other words, each release embeds almost all its dependencies, including a Python iterpreter and several other packages. In practice, this means that you can have other Python environments already installed in your computer and they won't conflict with the ones used for the LSST software releases. + +To summarize, this mechanism may be convenient if you **don't want to regularly install or update the LSST software on your computer**: when you need it, you access online the release of your choice among the ones available at that time. + +================= +Intended Audience +================= + +This mechanism of distributing the LSST software, which supplements the `other mechanisms `_ provided by the project, is intended in particular for **individuals** willing to use the LSST software on their personnal computers. + +However, it is not limited to that use case. The same mechanism is used for making the LSST software available to the computers in the CC-IN2P3's `login `_ and `batch `_ farms. This is considered beneficial for reproducibility purposes, since as a scientist you can use exactly the same version of the LSST software on your personal computer, as well as in your notebooks and in your batch jobs executing at CC-IN2P3. + + +=================== +Supported Platforms +=================== + +The `reference platform `_ for the LSST science pipelines software is currently CentOS 7. Besides that plaform, the software has been shown to work on macOS and Ubuntu, among others systems. + +The distribution mechanism documented here has been tested on CentOS, Ubuntu and macOS but is expected to work on other Linux distributions as well. If you successfully test it on your favorite Linux distribution, please let us know (see :ref:`help`). + + +================== +How to Get Started +================== + +You are encouraged to visit the section :ref:`usage` to get more details to help you decide whether this distribution mechanism is convenient for your individual use case. If you think it is, you can take a few minutes to install and configure the required software (see :ref:`installation`). diff --git a/docs/usage.rst b/docs/usage.rst new file mode 100644 index 0000000..27eb036 --- /dev/null +++ b/docs/usage.rst @@ -0,0 +1,177 @@ +.. _usage: + +************************************************* +Usage of the Software under ``/cvmfs/sw.lsst.eu`` +************************************************* + +Once the CernVM-FS client is installed and configured in your computer, a one-time process (see :ref:`installation`), you can start using the LSST software. + +In this section you will find information on how to use the online LSST software repository. It appears in your computer **in read-only** mode under the path ``/cvmfs/sw.lsst.eu``. Super-user privileges are not required to access the files there in. + +Repository Layout +================= + +The namespace under ``/cvmfs/sw.lsst.eu`` is meant to be self-explanatory. There you will find a sub-directory per supported platform (i.e. ``darwin-x86_64``, ``linux-x86_64``), a subdirectory for each distribution (e.g. ``lsst_distrib``) and a subdirectory for each available release (e.g. ``v15.0``, ``w_2018_19``). It looks like: + +.. code-block:: bash + + $ tree -L 3 /cvmfs/sw.lsst.eu + /cvmfs/sw.lsst.eu + |-- darwin-x86_64 + | `-- lsst_distrib + | |-- v15.0 + | |-- w_2018_14 + | |-- w_2018_15 + | |-- w_2018_16 + | |-- w_2018_17 + | |-- w_2018_18 + | `-- w_2018_19 + `-- linux-x86_64 + `-- lsst_distrib + |-- v15.0 + |-- w_2018_14 + |-- w_2018_15 + |-- w_2018_16 + |-- w_2018_17 + |-- w_2018_18 + `-- w_2018_19 + + +Names of directories containing **stable releases** start with letter "v" (e.g. ``v15.0``) and directories where **weekly releases** are located are named starting with letter "w" (e.g. ``w_2018_19``). ``lsst_distrib`` is the name of the LSST distribution, that is, a coherent set of packages that together form the LSST science pipelines. Each release of the LSST software is built specifically for delivery via CernVM-FS according to the `official instructions `_. + +Each release of the LSST software you will find under ``/cvmfs/sw.lsst.eu``, be it stable or weekly, is mostly self contained: it includes its own EUPS (see below), its own **Python 3** distribution (typically `miniconda `_) and its own set of external packages that specific release depends on (e.g. ``numpy``, ``cfitsio``, etc.). In particular, since the Python distribution installed with each release includes its own interpreter, each release is independent and configured so **it does not conflict with other Python interpreter** you may have already installed on your computer. + +.. important:: + + The LSST science pipelines depend on the runtime libraries of the C++ compiler. **You must have the appropriate version of the C++ compiler installed on your computer** for the LSST software to execute properly, since that compiler is not included in ``/cvmfs/sw.lsst.eu``. + + It is likely that successive versions of the C++ compilers keep a backwards-compatible Application Binary Interface (ABI). In practice, that means that, in general, a more recent version of the C++ compiler than the one used to build the LSST software can be used. + + You can find the specifics of the C++ compiler a particular release of the LSST software depends on in the file ``README.txt`` in each release's top directory. + +Basic Usage +=========== + +The first step for using the LSST science pipelines is to select the release you want to use and bootstrap your environment for that specific release. For instance, to use LSST ``v15.0`` on a Linux computer do: + +.. code-block:: bash + + # Open a new terminal session using a BASH shell + $ source /cvmfs/sw.lsst.eu/linux-x86_64/lsst_distrib/v15.0/loadLSST.bash + +As a result of executing this command, some environmental variables are extended or initialized, such as ``PATH``, ``PYTHONPATH``, ``LD_LIBRARY_PATH`` and ``EUPS_PATH``. + +.. note:: + + Most of the recent releases of the LSST pipelines use GCC v6.3 on Linux, which is usually installed on CentOS and RedHat via the `devltoool-6 `_ package. If that package is installed on your computer, it will be automatically detected and activated as a result of the command above. + +The LSST software uses `EUPS `_ for managing the set of software products which are part of a given release. EUPS allows you to select the packages you want to use in a work session. For instance, to use the command line tasks for processing CFHT images, you would do: + +.. code-block:: bash + + $ setup obs_cfht + $ setup pipe_tasks + +After these steps, your working environment is modified so that you can use the command line tasks (e.g. ``ingestImages.py``, ``processCcd.py``, etc.) and import LSST-specific modules in your own Python programs (e.g. ``import lsst.daf.persistence``): + +.. code-block:: bash + + $ processCcd.py --help + usage: processCcd.py input [options] + + positional arguments: + input path to input data repository, relative to + $PIPE_INPUT_ROOT + + optional arguments: + -h, --help show this help message and exit + --calib RAWCALIB path to input calibration repository, relative to + $PIPE_CALIB_ROOT + ... + + +If later on you need to work with a different release, say weekly ``w_2018_19``, **you must create a new terminal session** and configure your environment for the that specific release. For instance: + +.. code-block:: bash + + # In a new terminal session with BASH shell + $ source /cvmfs/sw.lsst.eu/linux-x86_64/lsst_distrib/w_2018_19/loadLSST.bash + + # From this point on, your environment is set up to use release w_2018_19 + +At this point you may want to `run the LSST demo `_ and read the tutorials on `how to use the LSST Science Pipelines `_. + + +Advanced Usage +============== + +As presented above, each installed release includes its own miniconda Python distribution with a strict set of packages the LSST science pipelines depend on. For your convenience, a set of packages is added without modifying the dependencies of the LSST software. + +You can determine which version of the Python interpreter is used for a given release of the LSST stack and obtain the list of installed packages via the ``conda`` command. For instance, when using ``w_2018_19`` on macOS you get: + +.. code-block:: bash + + $ source /cvmfs/sw.lsst.eu/darwin-x86_64/lsst_distrib/w_2018_19/loadLSST.bash + + $ which python + /cvmfs/sw.lsst.eu/darwin-x86_64/lsst_distrib/w_2018_19/python/miniconda3-4.3.21/bin/python + + $ python --version + Python 3.6.2 :: Continuum Analytics, Inc. + + $ conda list + # packages in environment at /cvmfs/sw.lsst.eu/darwin-x86_64/lsst_distrib/w_2018_19/python/miniconda3-4.3.21: + # + appnope 0.1.0 py36hf537a9a_0 + asn1crypto 0.22.0 py36_0 + astropy 2.0.1 np113py36_0 + ... + yaml 0.1.6 0 + zeromq 4.2.5 h378b8a2_0 + zlib 1.2.8 3 + +Among the packages deliberately added to each installed release, there are the ones necessary to use the `Jupyter `_ interactive computing environment with Python 3. You can therefore launch Jupyter via one of the commands: + +.. code-block:: bash + + $ jupyter nootebook + +or + +.. code-block:: bash + + $ jupyter lab + +and you will get an LSST-enabled notebook environment ready to use. + +In a similar way to ``conda``, you can retrieve the list of EUPS-managed products included in a bootstraped release of the LSST software via the command: + +.. code-block:: bash + + $ eups list --name + afw + apr + apr_util + ... + wcslib + ws4py + xpa + +Then you can activate one of those products, for example: + +.. code-block:: bash + + $ setup obs_subaru + +More information about EUPS can be found in this `EUPS tutorial `_. + + +More Advanced Usage +=================== + +.. todo:: + + Add information about: + + * How to install additional packages by creating a custom conda environment + * How to install additional EUPS products on top of a read-only installation diff --git a/etc/cvmfs/domain.d/lsst.eu.conf b/etc/cvmfs/domain.d/lsst.eu.conf new file mode 100644 index 0000000..d6684b9 --- /dev/null +++ b/etc/cvmfs/domain.d/lsst.eu.conf @@ -0,0 +1,27 @@ +#-----------------------------------------------------------------------------# +# File: /etc/cvmfs/domain.d/lsst.eu.conf # +# Purpose: configuration of CernVM-FS file system 'sw.lsst.eu' # +# Author: Fabio Hernandez (fabio@in2p3.fr) # +# Documentation: https://sw.lsst.eu # +#-----------------------------------------------------------------------------# + + +# +# List of stratum 0 servers of the repositories for the 'lsst.eu' domain +# Separate servers by ';' +# +CVMFS_SERVER_URL='http://cclssts1.in2p3.fr/cvmfs/@fqrn@' + + +# +# List of load-balanced proxy groups. Each proxy group is composed +# of a list of URLs separated by ';'. Groups are separated by '|'. +# +CVMFS_HTTP_PROXY='http://cclsstsqd.in2p3.fr:3128' + + +# +# Directory where the public key for the repositories of the domain +# 'lsst.eu' is located. +# +CVMFS_KEYS_DIR='/etc/cvmfs/keys/lsst.eu' diff --git a/etc/cvmfs/domain.d/lsst.eu.local b/etc/cvmfs/domain.d/lsst.eu.local new file mode 100644 index 0000000..cf03f4e --- /dev/null +++ b/etc/cvmfs/domain.d/lsst.eu.local @@ -0,0 +1,45 @@ +#-----------------------------------------------------------------------------# +# File: /etc/cvmfs/domain.d/lsst.eu.local # +# Purpose: configuration of CernVM-FS file system 'sw.lsst.eu' # +# Author: Fabio Hernandez (fabio@in2p3.fr) # +# Documentation: https://sw.lsst.eu # +#-----------------------------------------------------------------------------# + + +# +# If needed, add 'sw.lsst.eu' to the list of repositories this computer +# can mount +# +CVMFS_REPOSITORIES=${CVMFS_REPOSITORIES:-sw.lsst.eu} +if [[ ! $CVMFS_REPOSITORIES =~ sw.lsst.eu ]]; then + CVMFS_REPOSITORIES=${CVMFS_REPOSITORIES}',sw.lsst.eu' +fi + + +# +# Amount of local storage (in megabytes) the CernVM-FS client will +# use to cache remote files in local storage of this computer. +# A single version of the LSST software stack and its dependencies +# takes more than 10 GB, but not all files in a given version are +# used at the same time. +# +CVMFS_QUOTA_LIMIT=${CVMFS_QUOTA_LIMIT:-10000} +if [[ $CVMFS_QUOTA_LIMIT -le 10000 ]]; then + CVMFS_QUOTA_LIMIT=10000 +fi + + +# +# This tells the CernVM-FS client to use a single local cache for +# all the repositories mounted by this computer. Set to 'yes' if +# not already set. +# +CVMFS_SHARED_CACHE=${CVMFS_SHARED_CACHE:-yes} + + +# +# The default cache directory is '/var/lib/cvmfs' but you can change it by +# uncommenting and customizing the line below. Make sure this directory +# exists and is located on a local file system. +# +# CVMFS_CACHE_BASE=/path/to/my/cache diff --git a/etc/cvmfs/keys/lsst.eu/lsst.eu.pub b/etc/cvmfs/keys/lsst.eu/lsst.eu.pub new file mode 100644 index 0000000..c1eb575 --- /dev/null +++ b/etc/cvmfs/keys/lsst.eu/lsst.eu.pub @@ -0,0 +1,9 @@ +-----BEGIN PUBLIC KEY----- +MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAxFLu9yNfPH80Z/wOqN3Z +t8CYiw610vATFbntUXq1OYCVlDq1lYKykV5QJcNtlfj4QSAV3Z9gp/jlELjo8p4W +CIGXJAIri0f+W5ii1py1l2CdkXpKEyMw9z91Pi3Wcv/we6ahkHC8wNHQZNKnTDRW +8Hyye8VXx1bdZ6srqU8t9W++WWcQedQaykQNo2EDm1EUyzpGUyPMxLkOxW/K+/0h +bCZOyyfukhyANRt+dMIQwrVTb05GVDdyTckEHC+QDJJSokhbiOl0L58d9ZQ53OG7 +aJ8P/9kDZ3LE4Bcboa8liaDbHUiLroMHuvcJNOnEQzPW4MavNsoR8jYiyqd3/Ob5 +dwIDAQAB +-----END PUBLIC KEY----- diff --git a/tests/centos/Dockerfile b/tests/centos/Dockerfile new file mode 100644 index 0000000..7aaeaec --- /dev/null +++ b/tests/centos/Dockerfile @@ -0,0 +1,21 @@ +FROM centos:centos7 + +# +# Install CERN RPM repository and cvmfs client +# +RUN yum install -q -y https://ecsft.cern.ch/dist/cvmfs/cvmfs-release/cvmfs-release-latest.noarch.rpm \ + && yum install -q -y cvmfs + +# +# Install convenience tools +# +RUN yum install -q -y tree + +# +# Add the RPM packages +# +ADD ["./*.rpm", "./"] + +RUN yum install -q -y *.rpm + +ENTRYPOINT /bin/bash diff --git a/tests/centos/Makefile b/tests/centos/Makefile new file mode 100644 index 0000000..a1ae967 --- /dev/null +++ b/tests/centos/Makefile @@ -0,0 +1,10 @@ +.PHONY: all + +all: run + +run: pkgs + docker build -t airnandez/sw-lsst-eu-centos-test . + docker run -it airnandez/sw-lsst-eu-centos-test + +pkgs: + @cp ../../*.rpm . \ No newline at end of file diff --git a/tests/ubuntu/Dockerfile b/tests/ubuntu/Dockerfile new file mode 100644 index 0000000..85195f0 --- /dev/null +++ b/tests/ubuntu/Dockerfile @@ -0,0 +1,25 @@ +FROM ubuntu:16.04 + +RUN apt-get update + +# +# Install CERN RPM repository and cvmfs client +# +RUN apt-get update \ + && apt-get --assume-yes install lsb-release \ + && apt-get --assume-yes install curl \ + && apt-get --assume-yes install tree + +RUN curl -s -o /tmp/cvmfs-release-latest_all.deb https://ecsft.cern.ch/dist/cvmfs/cvmfs-release/cvmfs-release-latest_all.deb \ + && dpkg -i /tmp/cvmfs-release-latest_all.deb \ + && apt-get update + +RUN apt-get --yes install cvmfs + +# +# Install DEB package +# +ADD ["./*.deb", "./"] +RUN dpkg -i *.deb + +ENTRYPOINT /bin/bash diff --git a/tests/ubuntu/Makefile b/tests/ubuntu/Makefile new file mode 100644 index 0000000..d9791a0 --- /dev/null +++ b/tests/ubuntu/Makefile @@ -0,0 +1,10 @@ +.PHONY: all + +all: run + +run: pkgs + docker build -t airnandez/sw-lsst-eu-ubuntu-test . + docker run -it airnandez/sw-lsst-eu-ubuntu-test + +pkgs: + @cp ../../*.deb . \ No newline at end of file diff --git a/version.sh b/version.sh new file mode 100644 index 0000000..30afd5b --- /dev/null +++ b/version.sh @@ -0,0 +1,4 @@ +# Returns the version of the configuration package +pkgVersion() { + echo "0.5" +}