Cgi to go

.github/workflows/ci.yaml

@@ -8,7 +8,7 @@ name: CI
 jobs:
   build:
     name: Build server image
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     services:
       postgres:
         image: postgres:15
@@ -43,7 +43,7 @@ jobs:
           VERSION="$VERSION-$GITHUB_REF_NAME"
         fi
         echo "version=$VERSION"
-        docker build --file ci/docker/api_Dockerfile --tag nivlheim:latest --build-arg version=$VERSION server/service
+        docker build --file ci/docker/api_Dockerfile --tag nivlheim:latest --build-arg version=$VERSION .
     - name: Docker save
       run: docker save nivlheim | gzip > nivlheim-image.tar.gz
     - name: Upload artifact
@@ -54,7 +54,7 @@ jobs:
 
   buildwww:
     name: Build httpd+cgi image
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     steps:
     - name: Checkout
       uses: actions/checkout@v4
@@ -73,7 +73,7 @@ jobs:
 
   buildclient:
     name: Build client image
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     steps:
     - name: Checkout
       uses: actions/checkout@v4
@@ -93,7 +93,7 @@ jobs:
   test-scripts:
     name: Run test scripts
     needs: [build, buildwww, buildclient]
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     strategy:
       matrix:
         test:
@@ -115,7 +115,7 @@ jobs:
         docker load --input nivlheim-image.tar.gz/nivlheim-image.tar.gz
         docker load --input nivlheim-client.tar.gz/nivlheim-client.tar.gz
     - name: Start containers
-      run: docker-compose -f "ci/docker/docker-compose.yml" up -d
+      run: docker compose -f "ci/docker/docker-compose.yml" up -d
     - name: SSL handshake
       run: openssl s_client -connect localhost:443 -prexit
       continue-on-error: true
@@ -125,25 +125,25 @@ jobs:
       run: tests/test_${{ matrix.test }}
     - name: Retrieve logs from Docker
       if: always()
-      run: docker-compose -f "ci/docker/docker-compose.yml" logs
+      run: docker compose -f "ci/docker/docker-compose.yml" logs
     - name: Retrieve server logs
       if: always()
       run: |
         echo "------- access_log -------------------------------"
-        docker exec docker_nivlheimweb_1 grep -v 127.0.0.1 /var/log/httpd/access_log  || true
+        docker exec docker-nivlheimweb-1 grep -v 127.0.0.1 /var/log/httpd/access_log  || true
         echo "------- error_log --------------------------------"
-        docker exec docker_nivlheimweb_1 cat /var/log/httpd/error_log || true
+        docker exec docker-nivlheimweb-1 cat /var/log/httpd/error_log || true
         echo "------- system.log--------------------------------"
-        docker exec docker_nivlheimweb_1 cat /var/log/nivlheim/system.log || true
+        docker exec docker-nivlheimweb-1 cat /var/log/nivlheim/system.log || true
     - name: Stop containers
       if: always()
-      run: docker-compose -f "ci/docker/docker-compose.yml" down
+      run: docker compose -f "ci/docker/docker-compose.yml" down
 
   publish:
     if: ${{ github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/tags/') }}
     name: Publish server image
     needs: [test-scripts]
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     permissions:
       packages: write
       contents: read
@@ -173,7 +173,7 @@ jobs:
     if: ${{ github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/tags/') }}
     name: Publish httpd+cgi image
     needs: [test-scripts]
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     permissions:
       packages: write
       contents: read
@@ -212,7 +212,7 @@ jobs:
     if: ${{ github.ref == 'refs/heads/master' }}
     name: Tag and release
     needs: [test-scripts]
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
         with:

VERSION

@@ -1 +1 @@
-2.7.23
+2.7.24

ci/docker/Dockerfile

@@ -1,36 +1,22 @@
 FROM fedora:latest
-MAINTAINER iti-dia@usit.uio.no
 EXPOSE 80 443
 LABEL no.uio.contact=usit-gid@usit.uio.no
-LABEL description="Web server with CGI scripts for Nivlheim"
+LABEL description="Web server for Nivlheim"
 ARG BRANCH=""
 
-RUN dnf install -y httpd mod_ssl perl procps-ng \
+RUN dnf install -y httpd mod_ssl procps-ng \
 		unzip file bind-utils npm \
-		perl-Archive-Tar perl-Archive-Zip perl-CGI perl-Crypt-OpenSSL-X509 \
-		perl-DBD-Pg perl-DBI perl-Digest-CRC perl-Encode \
-		perl-File-Basename perl-File-Path perl-File-Temp \
-		perl-JSON perl-Log-Log4perl perl-Log-Dispatch perl-Log-Dispatch-FileRotate \
-		perl-Net-CIDR perl-Net-DNS perl-Net-IP \
-		perl-Proc-PID-File perl-Time-Piece \
 	&& dnf install -y --releasever=39 openssl \
 	&& dnf clean all \
 	&& rm -rf /var/cache/yum \
 	&& npm install -g handlebars
 
 # config
-COPY server/openssl_ca.conf server/log4perl.conf /etc/nivlheim/
+COPY server/openssl_ca.conf /etc/nivlheim/
 COPY server/client_CA_cert.sh /usr/bin/
 COPY server/httpd_ssl.conf /etc/httpd/conf.d/ssl.conf
 COPY server/httpd.conf /etc/httpd/conf/httpd.conf
 
-# cgi scripts
-COPY server/cgi/ping server/cgi/reqcert server/cgi/processarchive /var/www/cgi-bin/
-COPY server/cgi/ping2 /var/www/cgi-bin/secure/ping
-COPY server/cgi/post server/cgi/renewcert /var/www/cgi-bin/secure/
-COPY server/cgi/Database.pm /usr/lib64/perl5/Nivlheim/
-RUN chmod -R a+x /var/www/cgi-bin/*
-
 # copy the static web content
 COPY server/website /var/www/html/
 RUN rm -rf /var/www/html/mockapi

ci/docker/api_Dockerfile

@@ -4,12 +4,12 @@ ARG version
 
 WORKDIR /app
 
-COPY go.mod go.sum ./
+COPY server/service/go.mod server/service/go.sum ./
 RUN go mod download
 
-COPY *.go ./
-COPY ./database ./database
-COPY ./utility ./utility
+COPY server/service/*.go ./
+COPY server/service/database ./database
+COPY server/service/utility ./utility
 
 RUN go build -o /nivlheim -ldflags "-X main.version=${version:-UNDEFINED}"
 
@@ -25,5 +25,6 @@ WORKDIR /
 
 RUN apt-get update -qq && apt-get install -yqq ca-certificates
 COPY --from=build /nivlheim /nivlheim
+COPY server/server.conf /etc/nivlheim/server.conf
 
 ENTRYPOINT ["/nivlheim"]

client/nivlheim_client

@@ -67,7 +67,7 @@ my $NAME    = 'nivlheim_client';
 my $AUTHOR  = 'Øyvind Hagberg';
 my $CONTACT = 'oyvind.hagberg@usit.uio.no';
 my $RIGHTS  = 'USIT/IT-DRIFT/GD/GID, University of Oslo, Norway';
-my $VERSION = '2.7.23';
+my $VERSION = '2.7.24';
 
 # Usage text
 my $USAGE = <<"END_USAGE";

client/windows/nivlheim_client.ps1

@@ -31,7 +31,7 @@ param(
 	[bool]$nosleep = $false
 )
 
-Set-Variable version -option Constant -value "2.7.23"
+Set-Variable version -option Constant -value "2.7.24"
 Set-Variable useragent -option Constant -value "NivlheimPowershellClient/$version"
 Set-PSDebug -strict
 Set-StrictMode -version "Latest"	# http://technet.microsoft.com/en-us/library/hh849692.aspx

debian/changelog

@@ -1,3 +1,9 @@
+nivlheim (2.7.24-1) buster; urgency=low
+
+  * Changes in the server code (cgi scripts rewritten in Go)
+
+ -- Øyvind Hagberg <oyvind.hagberg@usit.uio.no>  Thu, 10 Oct 2024 09:30:00 +0200
+
 nivlheim (2.7.23-1) buster; urgency=low
 
   * Changes in the server code (Dependabot patches)

server/entrypoint.sh

@@ -7,7 +7,7 @@ if [ `whoami` != "root" ]; then
 fi
 
 # make dirs
-mkdir -p /var/www/nivlheim/{db,certs,CA,queue}
+mkdir -p /var/www/nivlheim/{db,certs,CA,queue,upload}
 mkdir -p /var/log/nivlheim
 mkdir -p /var/log/httpd
 

server/httpd_ssl.conf

@@ -22,16 +22,22 @@ SSLVerifyDepth  10
     SSLOptions +StdEnvVars
 </Files>
 
-<Directory "/var/www/cgi-bin">
+<Location "/cgi-bin/">
     SSLOptions +StdEnvVars
     SSLRequireSSL
-</Directory>
+    ProxyPass "http://nivlheimapi:4040/cgi-bin/"
+</Location>
 
-<Directory "/var/www/cgi-bin/secure">
-    SSLOptions +StdEnvVars +ExportCertData
+<Location "/cgi-bin/secure/">
     SSLRequireSSL
     SSLRequire %{SSL_CLIENT_VERIFY} eq "SUCCESS"
-</Directory>
+    RequestHeader set Cert-Client-Cert "%{SSL_CLIENT_CERT}s"
+    RequestHeader set Cert-Client-V-Remain "%{SSL_CLIENT_V_REMAIN}s"
+    RequestHeader set Cert-Client-S-DN "%{SSL_CLIENT_S_DN}s"
+    RequestHeader set Cert-Client-I-DN "%{SSL_CLIENT_I_DN}s"
+    RequestHeader set Cert-Client-S-DN-CN "%{SSL_CLIENT_S_DN_CN}s"
+    ProxyPass "http://nivlheimapi:4040/cgi-bin/secure/"
+</Location>
 
 <Location "/api/">
     ProxyPass "http://nivlheimapi:4040/api/"

server/server.conf

@@ -16,11 +16,16 @@ LDAPprimaryAttr=
 LDAPadminGroup=
 AllAccessGroups=
 HostOwnerPluginURL=
-CFEngineKeyDir=
+CFEngineKeyDir=/var/cfekeys
 PGhost=
 PGport=
 PGdatabase=
 PGuser=
 PGpassword=
 PGsslmode=
 HTTPListenAddress=
+CACertFile=CA/nivlheimca.crt
+CAKeyFile=CA/nivlheimca.key
+ConfDir=/var/www/nivlheim
+QueueDir=/var/www/nivlheim/queue
+UploadDir=/var/www/nivlheim/upload

server/server.conf-example.txt

@@ -23,3 +23,8 @@ PGdatabase=nivlheimdb
 PGuser=someuser
 PGpassword=abcd1234
 PGsslmode=require
+CACertFile=CA/nivlheimca.crt
+CAKeyFile=CA/nivlheimca.key
+ConfDir=/var/www/nivlheim
+QueueDir=/var/www/nivlheim/queue
+UploadDir=/var/www/nivlheim/upload

server/service/api.go

@@ -69,8 +69,16 @@ func createAPImuxer(theDB *sql.DB, devmode bool) *http.ServeMux {
 	api.Handle("/api/v2/status", &apiMethodStatus{db: theDB})
 	api.HandleFunc("/api/v2/userinfo", apiGetUserInfo)
 
+	// called by the nivlheim client, ported from perl
+	api.HandleFunc("/cgi-bin/ping", apiPing)
+	api.Handle("/cgi-bin/reqcert", &apiMethodReqCert{db: theDB})
+	api.Handle("/cgi-bin/secure/renewcert", &apiMethodRenewCert{db: theDB})
+	api.Handle("/cgi-bin/secure/ping", &apiMethodSecurePing{db: theDB})
+	api.Handle("/cgi-bin/secure/post", &apiMethodPostArchive{db: theDB})
+
 	// Add CSRF protection to all the api functions
 	mux.Handle("/api/v2/", wrapCSRFprotection(api))
+	mux.Handle("/cgi-bin/", wrapCSRFprotection(api))
 
 	// Oauth2-related endpoints
 	mux.HandleFunc("/api/oauth2/start", startOauth2Login)
@@ -80,8 +88,6 @@ func createAPImuxer(theDB *sql.DB, devmode bool) *http.ServeMux {
 	// internal API functions. Only allowed from localhost.
 	internal := http.NewServeMux()
 	internal.HandleFunc("/api/internal/triggerJob/", runJob)
-	internal.HandleFunc("/api/internal/unsetCurrent", unsetCurrent)
-	internal.HandleFunc("/api/internal/countFiles", countFiles)
 	internal.HandleFunc("/api/internal/replaceCertificate", replaceCertificate)
 	mux.Handle("/api/internal/", wrapOnlyAllowLocal(internal))
 

server/service/api_internal.go

@@ -5,8 +5,6 @@ import (
 	"net/http"
 	"reflect"
 	"regexp"
-	"strconv"
-	"strings"
 )
 
 // runJob sets the "trigger" flag on the Job struct in the jobs array,
@@ -37,44 +35,6 @@ func runJob(w http.ResponseWriter, req *http.Request) {
 	http.Error(w, "Job not found.", http.StatusNotFound)
 }
 
-// unsetCurrent is an internal API function that the CGI scripts use
-// to notify the system service/daemon that some file(s) have had
-// their "current" flag cleared, and can be removed from the
-// in-memory search cache.
-func unsetCurrent(w http.ResponseWriter, req *http.Request) {
-	if !isLocal(req) {
-		http.Error(w, "Only local requests are allowed", http.StatusForbidden)
-		return
-	}
-	for _, s := range strings.Split(req.FormValue("ids"), ",") {
-		fileID, err := strconv.ParseInt(s, 10, 64)
-		if err == nil {
-			removeFileFromFastSearch(fileID)
-		}
-	}
-	http.Error(w, "OK", http.StatusNoContent)
-}
-
-// countFiles is an internal API function that the CGI scripts use
-// to notify the system service/daemon that a number of files
-// have been processed, so we can produce an accurate count of
-// files-per-minute.
-func countFiles(w http.ResponseWriter, req *http.Request) {
-	if !isLocal(req) {
-		http.Error(w, "Only local requests are allowed", http.StatusForbidden)
-		return
-	}
-	i, err := strconv.Atoi(req.FormValue("n"))
-	if err != nil {
-		http.Error(w, "Invalid number: "+req.FormValue("n"), http.StatusBadRequest)
-		return
-	}
-	if i > 0 {
-		pfib.Add(float64(i)) // pfib = parsed files interval buffer
-	}
-	http.Error(w, "OK", http.StatusNoContent)
-}
-
 func doNothing(w http.ResponseWriter, req *http.Request) {
 	w.Header().Set("Content-Type", "text/plain; charset=utf-8")
 	fmt.Fprintf(w, "無\n\n") // https://en.wikipedia.org/wiki/Mu_(negative)

server/service/api_status.go

@@ -74,8 +74,7 @@ func (vars *apiMethodStatus) ServeHTTP(w http.ResponseWriter, req *http.Request)
 	// IncomingQueueSize
 	// TODO optimize for large directories
 	status.IncomingQueueSize = -1
-	const queuedir = "/var/www/nivlheim/queue"
-	f, err := os.Open(queuedir)
+	f, err := os.Open(config.QueueDir)
 	if err == nil {
 		defer f.Close()
 		names, err := f.Readdirnames(0)