Update MELPA to build both unstable and stable. (melpa#5612)

- Combine stable and unstable builds into the same rotation so one
builds after the other.

- Move all the services to docker containers,
- nginx: contains everything for handling http traffic including log
  rotation and ssl certs.
- syncer: manages syncing all the html files to the latest build
  information.
- logprocessor: processes the http logs.
- rsyncd: rsync for melpa repo.

.gitignore

@@ -4,6 +4,8 @@
 /.cask
 /.ecukes-failing-scenarios
 /config.mk
+/download_log_stable.db
+/download_log_stable.json.gz
 /download_log.db
 /download_log.json.gz
 /elpa
@@ -14,6 +16,12 @@
 /html/download_counts.json
 /html/recipes.json
 /html/updates.rss
+/html-stable/.last-build-duration
+/html-stable/archive.json
+/html-stable/build-status.json
+/html-stable/download_counts.json
+/html-stable/recipes.json
+/html-stable/updates.rss
 /packages-stable/*
 /packages/*
 /process_log.pid

Makefile

@@ -9,10 +9,13 @@ PKGDIR  := packages
 RCPDIR  := recipes
 HTMLDIR := html
 WORKDIR := working
-WEBROOT := $$HOME/www
 SLEEP   ?= 0
 SANDBOX := sandbox
 STABLE  ?= nil
+ifneq ($(STABLE), nil)
+PKGDIR  := packages-stable
+HTMLDIR := html-stable
+endif
 
 LISP_CONFIG ?= '(progn\
   (setq package-build-working-dir "$(TOP)/$(WORKDIR)/")\
@@ -51,7 +54,7 @@ clean-packages:
 
 clean-json:
 	@echo " • Removing json files ..."
-	@-rm -vf html/archive.json html/recipes.json
+	@-rm -vf $(HTMLDIR)/archive.json $(HTMLDIR)/recipes.json
 
 clean-sandbox:
 	@echo " • Removing sandbox files ..."
@@ -60,13 +63,6 @@ clean-sandbox:
 		rmdir '$(SANDBOX)'; \
 	fi
 
-sync:
-	@echo " • Synchronizing files ..."
-	@rsync -avz --delete $(PKGDIR)/ $(WEBROOT)/packages
-	@rsync -avz --safe-links --delete $(HTMLDIR)/* $(WEBROOT)/
-	@chmod -R go+rx $(WEBROOT)/packages/*
-
-
 pull-package-build:
 	git subtree pull --squash -P package-build package-build master
 
@@ -93,7 +89,15 @@ html/recipes.json: $(RCPDIR)/.dirstamp
 	@echo " • Building $@ ..."
 	@$(EVAL) '(package-build-recipe-alist-as-json "html/recipes.json")'
 
-json: html/archive.json html/recipes.json
+html-stable/archive.json: $(PKGDIR)/archive-contents
+	@echo " • Building $@ ..."
+	@$(EVAL) '(package-build-archive-alist-as-json "html-stable/archive.json")'
+
+html-stable/recipes.json: $(RCPDIR)/.dirstamp
+	@echo " • Building $@ ..."
+	@$(EVAL) '(package-build-recipe-alist-as-json "html-stable/recipes.json")'
+
+json: $(HTMLDIR)/archive.json $(HTMLDIR)/recipes.json
 
 $(RCPDIR)/.dirstamp: .FORCE
 	@[[ ! -e $@ || "$$(find $(@D) -newer $@ -print -quit)" != "" ]] \

docker/builder/Dockerfile

@@ -0,0 +1,5 @@
+FROM debian:unstable
+RUN apt-get update
+RUN DEBIAN_FRONTEND=noninteractive apt-get --yes install emacs25-nox git make mercurial ruby
+WORKDIR /mnt/store/melpa
+CMD docker/builder/run.sh

scripts/parallel_build_all → docker/builder/parallel_build_all

@@ -14,7 +14,7 @@ function build_all {
     PATH=$HOME/.cabal/bin:$HOME/usr/bin:$HOME/bin:$PATH
 
     ## run the script
-    cd "$(dirname "${0}")/.."
+    cd "/mnt/store/melpa"
     make -j4 $(grep --files-without-match ':fetcher\s*wiki' $(find recipes/ -type f -not -name python-info -and -not -name python3-info | sort)) &
     wait
 

docker/builder/run.sh

@@ -0,0 +1,72 @@
+#!/bin/bash -e
+
+MELPA_REPO=/mnt/store/melpa
+cd "${MELPA_REPO}"
+MELPA_BRANCH=$( git rev-parse --abbrev-ref HEAD )
+
+STATUS_JSON=${MELPA_REPO}/html/build-status.json
+LAST_DURATION_FILE=${MELPA_REPO}/html/.last-build-duration
+STABLE_STATUS_JSON=${MELPA_REPO}/html-stable/build-status.json
+STABLE_LAST_DURATION_FILE=${MELPA_REPO}/html-stable/.last-build-duration
+
+## update MELPA repo
+git fetch origin
+git reset --hard "origin/${MELPA_BRANCH}"
+git pull origin "${MELPA_BRANCH}"
+echo
+
+update_json() {
+    cat <<EOF > $BUILD_STATUS_JSON
+{
+  "started": $BUILD_STARTED,
+  "completed": ${BUILD_COMPLETED-null},
+  "next": ${BUILD_NEXT-null},
+  "duration": ${BUILD_DURATION-null}
+}
+EOF
+    echo "Writing $BUILD_STATUS_JSON"
+    cat "$BUILD_STATUS_JSON"
+}
+
+flux_capacitor() {
+
+    if [ -f "$BUILD_LAST_DURATION_FILE" ]; then
+        BUILD_DURATION=$(cat "$BUILD_LAST_DURATION_FILE")
+    fi
+
+
+    BUILD_STARTED=$(date "+%s")
+    update_json
+
+    # Build all the packages.
+    docker/builder/parallel_build_all
+
+    # Store completed date
+    BUILD_COMPLETED=$(date "+%s")
+    BUILD_DURATION=$((BUILD_COMPLETED - BUILD_STARTED))
+    echo -n "$BUILD_DURATION" > $BUILD_LAST_DURATION_FILE
+    BUILD_NEXT=$((BUILD_COMPLETED + BUILD_DELAY))
+    update_json
+
+}
+
+unset STABLE
+
+BUILD_STATUS_JSON=${STATUS_JSON}
+BUILD_LAST_DURATION_FILE=${LAST_DURATION_FILE}
+if [ -f "$STABLE_LAST_DURATION_FILE" ]; then
+    BUILD_DELAY=$(cat "$STABLE_LAST_DURATION_FILE")
+fi
+
+flux_capacitor
+
+# stable build
+export STABLE=t
+
+BUILD_STATUS_JSON=${STABLE_STATUS_JSON}
+BUILD_LAST_DURATION_FILE=${STABLE_LAST_DURATION_FILE}
+if [ -f "$LAST_DURATION_FILE" ]; then
+    BUILD_DELAY=$(cat "$LAST_DURATION_FILE")
+fi
+
+flux_capacitor

docker/docker-compose.yml

@@ -0,0 +1,44 @@
+version: "3"
+services:
+  nginx:
+    build: ./nginx
+    image: melpa/nginx:v1
+    restart: unless-stopped
+    volumes:
+      - /mnt/store/melpa:/mnt/store/melpa:ro
+      - /mnt/store/letsencrypt:/etc/letsencrypt
+      - /mnt/store/log:/mnt/store/log
+      - /mnt/store/log-stable:/mnt/store/log-stable
+    ports:
+      - "80:80"
+      - "443:443"
+  builder:
+    build: ./builder
+    image: melpa/builder:v1
+    restart: unless-stopped
+    volumes:
+      - /mnt/store/melpa:/mnt/store/melpa
+  logprocessor:
+    build: ./logprocessor
+    image: melpa/logprocessor:v1
+    restart: unless-stopped
+    volumes:
+      - /mnt/db:/mnt/db
+      - /mnt/store/melpa:/mnt/store/melpa
+      - /mnt/store/log:/mnt/store/log:ro
+      - /mnt/store/log-stable:/mnt/store/log-stable:ro
+  syncer:
+    build: ./syncer
+    image: melpa/syncer:v1
+    restart: unless-stopped
+    volumes:
+      - /mnt/store/melpa:/mnt/store/melpa
+  rsyncd:
+    build: ./rsyncd
+    image: melpa/rsyncd:v1
+    restart: unless-stopped
+    ports:
+      - "873:873"
+    volumes:
+      - /mnt/store/melpa/packages:/packages:ro
+      - /mnt/store/melpa/packages-stable:/packages-stable:ro

docker/logprocessor/Dockerfile

@@ -0,0 +1,5 @@
+FROM debian:unstable
+RUN apt-get update
+RUN DEBIAN_FRONTEND=noninteractive apt-get --yes install python
+WORKDIR /mnt/store/melpa
+CMD docker/logprocessor/run.sh

scripts/process_log.py → docker/logprocessor/process_log.py

@@ -13,7 +13,10 @@
 import sqlite3
 from operator import or_
 
-LOGFILE = "/home/melpa/log/melpa.access.log"
+STABLE = os.getenv("STABLE")
+LOGFILE = "/mnt/store/log/melpa.access.log"
+if STABLE:
+    LOGFILE = "/mnt/store/log-stable/melpa.access.log"
 LOGREGEX = r'(?P<ip>[\d.]+) [ -]+ \[(?P<date>[\w/: +-]+)\] ' \
            r'"GET /packages/(?P<package>[^ ]+)-[0-9.]+.(?:el|tar) ' \
            r'HTTP/\d.\d" 200'
@@ -25,8 +28,8 @@ def json_handler(obj):
     elif isinstance(obj, set):
         return list(obj)
     raise TypeError(
-        'Object of type {0} with value {0} is not JSON serializable'.format(
-        type(obj), repr(obj)))
+        'Object of type {0} with value {1} is not JSON serializable'.format(
+            type(obj), repr(obj)))
 
 
 def json_dump(data, jsonfile, indent=None):
@@ -58,6 +61,7 @@ def ip_to_number(ip):
     return reduce(or_, ((int(n) << (i*8)) for i, n in enumerate(
         reversed(ip.split('.')))), 0)
 
+
 def parse_logfile(logfilename, curs):
     """
     """
@@ -77,9 +81,6 @@ def parse_logfile(logfilename, curs):
 
         # Convert ips to four character strings.
         ip = match.group('ip')
-        dtstamp = int(time.mktime(
-            datetime.strptime(match.group('date').split()[0],
-                              "%d/%b/%Y:%H:%M:%S").timetuple()))
         pkg = match.group('package')
 
         curs.execute("INSERT OR IGNORE INTO pkg_ip VALUES (?, ?)", (pkg, ip))
@@ -96,28 +97,17 @@ def main():
                         help="Log files to parse.", default=[LOGFILE])
     args = parser.parse_args()
 
-    pid = str(os.getpid())
-    pidfile = os.path.join(os.path.join(tempfile.gettempdir(), "process_log.pid"))
-
-    if os.access(pidfile, os.F_OK):
-        running_pid = open(pidfile, "r").readline()
-
-        try:
-            os.kill(int(running_pid), 0)
-            print "Process {0} currently running.".format(running_pid)
-            return 1
-        except OSError:
-            print "Stale lockfile."
-            os.unlink(pidfile)
+    db_filename = "/mnt/db/download_log.db"
+    if STABLE:
+        db_filename = "/mnt/db/download_log_stable.db"
 
-    file(pidfile, 'w').write(pid)
-
-    new_db = not os.path.exists("download_log.db")
-    conn = sqlite3.connect("download_log.db")
+    new_db = not os.path.exists(db_filename)
+    conn = sqlite3.connect(db_filename)
     curs = conn.cursor()
     if new_db:
         sys.stdout.write("creating database...\n")
-        curs.execute('''CREATE TABLE pkg_ip (package, ip, PRIMARY KEY (package, ip))''')
+        curs.execute(
+            '''CREATE TABLE pkg_ip (package, ip, PRIMARY KEY (package, ip))''')
         conn.commit()
 
     # parse each parameter
@@ -130,10 +120,14 @@ def main():
         conn.commit()
 
     # calculate current package totals
-    pkgcount = {p: c for p,c in curs.execute("SELECT package, count(ip) FROM pkg_ip GROUP BY 1")}
-    json_dump(pkgcount, open("html/download_counts.json", 'w'), indent=1)
+    pkgcount = {p: c for p, c in curs.execute(
+        "SELECT package, count(ip) FROM pkg_ip GROUP BY 1")}
+    if STABLE:
+        json_dump(pkgcount, open(
+            "html-stable/download_counts.json", 'w'), indent=1)
+    else:
+        json_dump(pkgcount, open("html/download_counts.json", 'w'), indent=1)
 
-    os.unlink(pidfile)
     return 0
 
 

docker/logprocessor/run.sh

@@ -0,0 +1,11 @@
+#!/bin/bash -e
+
+MELPA_REPO=/mnt/store/melpa
+cd "${MELPA_REPO}"
+
+unset STABLE
+/usr/bin/python ${MELPA_REPO}/docker/logprocessor/process_log.py
+
+export STABLE=t
+/usr/bin/python ${MELPA_REPO}/docker/logprocessor/process_log.py
+

docker/nginx/Dockerfile

@@ -0,0 +1,6 @@
+FROM nginx:alpine
+RUN apk add --no-cache certbot openssl logrotate
+COPY default.conf /etc/nginx/conf.d/
+COPY logrotate /etc/logrotate
+WORKDIR /mnt/store/melpa
+CMD docker/nginx/run.sh