Fix various things so the test scripts pass (#197)

* Ser om dette virker

* Change container names because of new docker compose

* Add a default server.conf to the container

* Fix openssl version (regression)

* Create the upload directory

* Modify test so it doesn't rely on cgi script file

* Make postArchive.go not care about content type

* Fix the cfengine test

* hostname without domain is no show stopper

* Fix a test that didn't use the correct content-type

* Bump version

* Remove unnecessary config from test_cfengine.sh

.github/workflows/ci.yaml

@@ -8,7 +8,7 @@ name: CI
 jobs:
   build:
     name: Build server image
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     services:
       postgres:
         image: postgres:15
@@ -43,7 +43,7 @@ jobs:
           VERSION="$VERSION-$GITHUB_REF_NAME"
         fi
         echo "version=$VERSION"
-        docker build --file ci/docker/api_Dockerfile --tag nivlheim:latest --build-arg version=$VERSION server/service
+        docker build --file ci/docker/api_Dockerfile --tag nivlheim:latest --build-arg version=$VERSION .
     - name: Docker save
       run: docker save nivlheim | gzip > nivlheim-image.tar.gz
     - name: Upload artifact
@@ -54,7 +54,7 @@ jobs:
 
   buildwww:
     name: Build httpd+cgi image
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     steps:
     - name: Checkout
       uses: actions/checkout@v4
@@ -73,7 +73,7 @@ jobs:
 
   buildclient:
     name: Build client image
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     steps:
     - name: Checkout
       uses: actions/checkout@v4
@@ -93,7 +93,7 @@ jobs:
   test-scripts:
     name: Run test scripts
     needs: [build, buildwww, buildclient]
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     strategy:
       matrix:
         test:
@@ -115,7 +115,7 @@ jobs:
         docker load --input nivlheim-image.tar.gz/nivlheim-image.tar.gz
         docker load --input nivlheim-client.tar.gz/nivlheim-client.tar.gz
     - name: Start containers
-      run: docker-compose -f "ci/docker/docker-compose.yml" up -d
+      run: docker compose -f "ci/docker/docker-compose.yml" up -d
     - name: SSL handshake
       run: openssl s_client -connect localhost:443 -prexit
       continue-on-error: true
@@ -125,25 +125,25 @@ jobs:
       run: tests/test_${{ matrix.test }}
     - name: Retrieve logs from Docker
       if: always()
-      run: docker-compose -f "ci/docker/docker-compose.yml" logs
+      run: docker compose -f "ci/docker/docker-compose.yml" logs
     - name: Retrieve server logs
       if: always()
       run: |
         echo "------- access_log -------------------------------"
-        docker exec docker_nivlheimweb_1 grep -v 127.0.0.1 /var/log/httpd/access_log  || true
+        docker exec docker-nivlheimweb-1 grep -v 127.0.0.1 /var/log/httpd/access_log  || true
         echo "------- error_log --------------------------------"
-        docker exec docker_nivlheimweb_1 cat /var/log/httpd/error_log || true
+        docker exec docker-nivlheimweb-1 cat /var/log/httpd/error_log || true
         echo "------- system.log--------------------------------"
-        docker exec docker_nivlheimweb_1 cat /var/log/nivlheim/system.log || true
+        docker exec docker-nivlheimweb-1 cat /var/log/nivlheim/system.log || true
     - name: Stop containers
       if: always()
-      run: docker-compose -f "ci/docker/docker-compose.yml" down
+      run: docker compose -f "ci/docker/docker-compose.yml" down
 
   publish:
     if: ${{ github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/tags/') }}
     name: Publish server image
     needs: [test-scripts]
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     permissions:
       packages: write
       contents: read
@@ -173,7 +173,7 @@ jobs:
     if: ${{ github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/tags/') }}
     name: Publish httpd+cgi image
     needs: [test-scripts]
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     permissions:
       packages: write
       contents: read
@@ -212,7 +212,7 @@ jobs:
     if: ${{ github.ref == 'refs/heads/master' }}
     name: Tag and release
     needs: [test-scripts]
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
         with:

VERSION

@@ -1 +1 @@
-2.7.23
+2.7.24

ci/docker/Dockerfile

@@ -4,8 +4,9 @@ LABEL no.uio.contact=usit-gid@usit.uio.no
 LABEL description="Web server for Nivlheim"
 ARG BRANCH=""
 
-RUN dnf install -y httpd mod_ssl openssl procps-ng \
+RUN dnf install -y httpd mod_ssl procps-ng \
 		unzip file bind-utils npm \
+	&& dnf install -y --releasever=39 openssl \
 	&& dnf clean all \
 	&& rm -rf /var/cache/yum \
 	&& npm install -g handlebars

ci/docker/api_Dockerfile

@@ -4,12 +4,12 @@ ARG version
 
 WORKDIR /app
 
-COPY go.mod go.sum ./
+COPY server/service/go.mod server/service/go.sum ./
 RUN go mod download
 
-COPY *.go ./
-COPY ./database ./database
-COPY ./utility ./utility
+COPY server/service/*.go ./
+COPY server/service/database ./database
+COPY server/service/utility ./utility
 
 RUN go build -o /nivlheim -ldflags "-X main.version=${version:-UNDEFINED}"
 
@@ -25,5 +25,6 @@ WORKDIR /
 
 RUN apt-get update -qq && apt-get install -yqq ca-certificates
 COPY --from=build /nivlheim /nivlheim
+COPY server/server.conf /etc/nivlheim/server.conf
 
 ENTRYPOINT ["/nivlheim"]

client/nivlheim_client

@@ -67,7 +67,7 @@ my $NAME    = 'nivlheim_client';
 my $AUTHOR  = 'Øyvind Hagberg';
 my $CONTACT = 'oyvind.hagberg@usit.uio.no';
 my $RIGHTS  = 'USIT/IT-DRIFT/GD/GID, University of Oslo, Norway';
-my $VERSION = '2.7.23';
+my $VERSION = '2.7.24';
 
 # Usage text
 my $USAGE = <<"END_USAGE";

client/windows/nivlheim_client.ps1

@@ -31,7 +31,7 @@ param(
 	[bool]$nosleep = $false
 )
 
-Set-Variable version -option Constant -value "2.7.23"
+Set-Variable version -option Constant -value "2.7.24"
 Set-Variable useragent -option Constant -value "NivlheimPowershellClient/$version"
 Set-PSDebug -strict
 Set-StrictMode -version "Latest"	# http://technet.microsoft.com/en-us/library/hh849692.aspx

debian/changelog

@@ -1,3 +1,9 @@
+nivlheim (2.7.24-1) buster; urgency=low
+
+  * Changes in the server code (cgi scripts rewritten in Go)
+
+ -- Øyvind Hagberg <oyvind.hagberg@usit.uio.no>  Thu, 10 Oct 2024 09:30:00 +0200
+
 nivlheim (2.7.23-1) buster; urgency=low
 
   * Changes in the server code (Dependabot patches)

server/entrypoint.sh

@@ -7,7 +7,7 @@ if [ `whoami` != "root" ]; then
 fi
 
 # make dirs
-mkdir -p /var/www/nivlheim/{db,certs,CA,queue}
+mkdir -p /var/www/nivlheim/{db,certs,CA,queue,upload}
 mkdir -p /var/log/nivlheim
 mkdir -p /var/log/httpd
 

server/server.conf

@@ -16,16 +16,16 @@ LDAPprimaryAttr=
 LDAPadminGroup=
 AllAccessGroups=
 HostOwnerPluginURL=
-CFEngineKeyDir=
+CFEngineKeyDir=/var/cfekeys
 PGhost=
 PGport=
 PGdatabase=
 PGuser=
 PGpassword=
 PGsslmode=
 HTTPListenAddress=
-CACertFile=
-CAKeyFile=
-ConfDir=
-QueueDir=
-UploadDir=
+CACertFile=CA/nivlheimca.crt
+CAKeyFile=CA/nivlheimca.key
+ConfDir=/var/www/nivlheim
+QueueDir=/var/www/nivlheim/queue
+UploadDir=/var/www/nivlheim/upload

server/service/postArchive.go

@@ -116,7 +116,7 @@ func (vars *apiMethodPostArchive) ServeHTTP(w http.ResponseWriter, req *http.Req
 
 	osHostName = strings.ToLower(osHostName)
 	shortHost := osHostName
-	match = regexp.MustCompile(`^(\S+?)\.`)
+	match = regexp.MustCompile(`^(\S+?)\..*$`)
 	shortHost2 := match.ReplaceAll([]byte(shortHost), []byte("$1"))
 
 	clientVersion := req.FormValue("version")
@@ -149,48 +149,32 @@ func (vars *apiMethodPostArchive) ServeHTTP(w http.ResponseWriter, req *http.Req
 
 	defer dst.Close()
 
-	if strings.HasPrefix(contentType, "multipart/form-data") {
-		if _, ok := req.MultipartForm.File["archive"]; ok {
-			rFile := "archive"
-			file, _, err := req.FormFile(rFile)
-
-			if err != nil {
-				http.Error(w, err.Error(), http.StatusBadRequest)
-				return
-			}
-
-			defer file.Close()
-
-			_, err = io.Copy(dst, file)
-
-			if err != nil {
-				log.Printf("Could not write archive file: %s", err.Error())
-				http.Error(w, err.Error(), http.StatusInternalServerError)
-				return
-			}
-
-		} else {
-			log.Printf("missing file upload parameter 'archive' (%s)", fingerprint)
-			http.Error(w, "File missing", http.StatusBadRequest)
+	if _, ok := req.MultipartForm.File["archive"]; ok {
+		rFile := "archive"
+		file, _, err := req.FormFile(rFile)
+		if err != nil {
+			http.Error(w, err.Error(), http.StatusBadRequest)
 			return
 		}
-	} else if strings.HasPrefix(contentType, "application/x-www-form-urlencoded") {
-		rFile := "archive_base64"
-		file := req.FormValue(rFile)
-		if file != "" {
-			decoder := base64.NewDecoder(base64.StdEncoding, strings.NewReader(file))
-			_, err = io.Copy(dst, decoder)
-			if err != nil {
-				log.Printf("Could not write archive file: %s", err.Error())
-				http.Error(w, err.Error(), http.StatusInternalServerError)
-				return
-			}
-
-		} else {
-			log.Printf("missing file upload parameter 'archive_base64' (%s)", fingerprint)
-			http.Error(w, "File missing", http.StatusBadRequest)
+		defer file.Close()
+		_, err = io.Copy(dst, file)
+		if err != nil {
+			log.Printf("Could not write archive file: %s", err.Error())
+			http.Error(w, err.Error(), http.StatusInternalServerError)
 			return
 		}
+	} else if file := req.FormValue("archive_base64"); file != "" {
+		decoder := base64.NewDecoder(base64.StdEncoding, strings.NewReader(file))
+		_, err = io.Copy(dst, decoder)
+		if err != nil {
+			log.Printf("Could not write archive file: %s", err.Error())
+			http.Error(w, err.Error(), http.StatusInternalServerError)
+			return
+		}
+	} else {
+		log.Printf("missing file upload parameter (%s)", fingerprint)
+		http.Error(w, "File missing", http.StatusBadRequest)
+		return
 	}
 
 	defer func() {
@@ -300,47 +284,32 @@ func (vars *apiMethodPostArchive) ServeHTTP(w http.ResponseWriter, req *http.Req
 
 	defer dst.Close()
 
-	if strings.HasPrefix(contentType, "multipart/form-data") {
-		if _, ok := req.MultipartForm.File["signature"]; ok {
-			rFile := "signature"
-			file, _, err := req.FormFile(rFile)
-
-			if err != nil {
-				http.Error(w, err.Error(), http.StatusBadRequest)
-				return
-			}
-
-			defer file.Close()
-
-			_, err = io.Copy(dst, file)
-
-			if err != nil {
-				log.Printf("Could not write archive file: %s", err.Error())
-				http.Error(w, err.Error(), http.StatusInternalServerError)
-				return
-			}
-
-		} else {
-			log.Printf("missing file upload parameter 'signature' (%s)", fingerprint)
-			http.Error(w, "File missing", http.StatusBadRequest)
+	if _, ok := req.MultipartForm.File["signature"]; ok {
+		rFile := "signature"
+		file, _, err := req.FormFile(rFile)
+		if err != nil {
+			http.Error(w, err.Error(), http.StatusBadRequest)
 			return
 		}
-	} else if strings.HasPrefix(contentType, "application/x-www-form-urlencoded") {
-		rFile := "signature_base64"
-		file := req.FormValue(rFile)
-		if file != "" {
-			decoder := base64.NewDecoder(base64.StdEncoding, strings.NewReader(file))
-			_, err = io.Copy(dst, decoder)
-			if err != nil {
-				log.Printf("Could not write signature file (%s): %s", fingerprint, err.Error())
-				http.Error(w, err.Error(), http.StatusInternalServerError)
-				return
-			}
-		} else {
-			log.Printf("missing file upload parameter 'signature_base64' (%s)", fingerprint)
-			http.Error(w, "File missing", http.StatusBadRequest)
+		defer file.Close()
+		_, err = io.Copy(dst, file)
+		if err != nil {
+			log.Printf("Could not write archive file: %s", err.Error())
+			http.Error(w, err.Error(), http.StatusInternalServerError)
 			return
 		}
+	} else if file := req.FormValue("signature_base64"); file != "" {
+		decoder := base64.NewDecoder(base64.StdEncoding, strings.NewReader(file))
+		_, err = io.Copy(dst, decoder)
+		if err != nil {
+			log.Printf("Could not write signature file (%s): %s", fingerprint, err.Error())
+			http.Error(w, err.Error(), http.StatusInternalServerError)
+			return
+		}
+	} else {
+		log.Printf("missing file upload parameter signature (%s)", fingerprint)
+		http.Error(w, "File missing", http.StatusBadRequest)
+		return
 	}
 
 	dstInfo, err = dst.Stat()

server/service/processarchive.go

@@ -101,23 +101,21 @@ func processArchive(url string, db *sql.DB) (err error) {
 	   / There's a bug in the Windows client, in some cases it gives the hostname without the domain.
 	   / See: https://github.com/unioslo/nivlheim/issues/138 */
 	if !strings.Contains(metaData["os_hostname"], ".") {
+		// The file might not exist. In that case, do nothing.
 		file, err := os.Open(tempDir + "/commands/DomainName")
-		if err != nil {
-			log.Printf("Could not open file %s: %s", tempDir+"/commands/DomainName", err)
-			return nil
-		}
-		defer file.Close()
-		scanner := bufio.NewScanner(file)
-		// first line is the command itself
-		scanner.Scan()
-		// second line is the output
-		scanner.Scan()
-		if err := scanner.Err(); err != nil {
-			return err
+		if err == nil {
+			defer file.Close()
+			scanner := bufio.NewScanner(file)
+			// first line is the command itself
+			scanner.Scan()
+			// second line is the output
+			scanner.Scan()
+			if err := scanner.Err(); err != nil {
+				return err
+			}
+			fqdn := metaData["hostname"] + "." + scanner.Text()
+			metaData["hostname"] = fqdn
 		}
-
-		fqdn := metaData["hostname"] + "." + scanner.Text()
-		metaData["hostname"] = fqdn
 	}
 
 	curFiles := make(map[string]int64)