t3c to use package metadata if rpm db is unhealthy (#7652)

* adding check to verify rpmdb

* added check to verify rpm db

* rebase and fix conflicts

* if rpmdb is unhealthy use package data in metadata

* requested changes addressed.

* added changelog entry

* fixed spelling error and added GoDoc commnet

* fixed formatting error

CHANGELOG.md

@@ -45,6 +45,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/).
 - [#7619](https://github.com/apache/trafficcontrol/pull/7619) Traffic Ops* added optional field `oauth_user_attribute` for OAuth login credentials
 - [#7641](https://github.com/apache/trafficcontrol/pull/7641) *Traffic Router* Added further optimization to TR's algorithm of figuring out the zone for an incoming request.
 - [#7646](https://github.com/apache/trafficcontrol/pull/7646) *Traffic Portal* Add the ability to delete a cert.
+- [#7652](https://github.com/apache/trafficcontrol/pull/7652) *t3c* added rpmdb checks and use package data from t3c-apply-metadata.json if rpmdb is corrupt
 
 ### Changed
 - [#7584](https://github.com/apache/trafficcontrol/pull/7584) *Documentation* Upgrade Traffic Control Sphinx documentation Makefile OS intelligent.

cache-config/t3c-apply/config/config.go

@@ -27,6 +27,7 @@ import (
 	"os"
 	"os/exec"
 	"path/filepath"
+	"regexp"
 	"strings"
 	"time"
 
@@ -81,6 +82,7 @@ type Cfg struct {
 	SvcManagement       SvcManagement
 	Retries             int
 	ReverseProxyDisable bool
+	RpmDBOk             bool
 	SkipOSCheck         bool
 	UseStrategies       t3cutil.UseStrategiesFlag
 	TOInsecure          bool
@@ -188,6 +190,29 @@ func directoryExists(dir string) (bool, os.FileInfo) {
 	return info.IsDir(), info
 }
 
+const rpmDir = "/var/lib/rpm"
+
+// verifies the rpm database files. if there is any database corruption
+// it will return false
+func verifyRpmDB() bool {
+	exclude := regexp.MustCompile(`(^\.|^__)`)
+	dbFiles, err := os.ReadDir(rpmDir)
+	if err != nil {
+		return false
+	}
+	for _, file := range dbFiles {
+		if exclude.Match([]byte(file.Name())) {
+			continue
+		}
+		cmd := exec.Command("/usr/lib/rpm/rpmdb_verify", rpmDir+"/"+file.Name())
+		err := cmd.Run()
+		if err != nil || cmd.ProcessState.ExitCode() > 0 {
+			return false
+		}
+	}
+	return true
+}
+
 // derives the ATS Installation directory from
 // the rpm config file list.
 func GetTSPackageHome() string {
@@ -322,10 +347,11 @@ If any of the related flags are also set, they override the mode's default behav
 	// so we want to log what flags the mode set here, to aid debugging.
 	// But we can't do that until the loggers are initialized.
 	modeLogStrs := []string{}
+	fatalLogStrs := []string{}
 	if getopt.IsSet(runModeFlagName) {
 		runMode := t3cutil.StrToMode(*runModePtr)
 		if runMode == t3cutil.ModeInvalid {
-			return Cfg{}, errors.New(*runModePtr + " is an invalid mode.")
+			fatalLogStrs = append(fatalLogStrs, *runModePtr+" is an invalid mode.")
 		}
 		modeLogStrs = append(modeLogStrs, "t3c-apply is running in "+runMode.String()+" mode")
 		switch runMode {
@@ -411,7 +437,7 @@ If any of the related flags are also set, they override the mode's default behav
 	}
 
 	if *verbosePtr > 2 {
-		return Cfg{}, errors.New("Too many verbose options. The maximum log verbosity level is 2 (-vv or --verbose=2) for errors (0), warnings (1), and info (2)")
+		fatalLogStrs = append(fatalLogStrs, "Too many verbose options. The maximum log verbosity level is 2 (-vv or --verbose=2) for errors (0), warnings (1), and info (2)")
 	}
 
 	var cacheHostName string
@@ -420,7 +446,7 @@ If any of the related flags are also set, they override the mode's default behav
 	} else {
 		cacheHostName, err = os.Hostname()
 		if err != nil {
-			return Cfg{}, errors.New("Could not get the hostname from the O.S., please supply a hostname: " + err.Error())
+			fatalLogStrs = append(fatalLogStrs, "Could not get the hostname from the O.S., please supply a hostname: "+err.Error())
 		}
 		// strings.Split always returns a slice with at least 1 element, so we don't need a len check
 		cacheHostName = strings.Split(cacheHostName, ".")[0]
@@ -429,7 +455,7 @@ If any of the related flags are also set, they override the mode's default behav
 	useGit := StrToUseGitFlag(*useGitStr)
 
 	if useGit == UseGitInvalid {
-		return Cfg{}, errors.New("Invalid git flag '" + *useGitStr + "'. Valid options are yes, no, auto.")
+		fatalLogStrs = append(fatalLogStrs, "Invalid git flag '"+*useGitStr+"'. Valid options are yes, no, auto.")
 	}
 
 	retries := *retriesPtr
@@ -471,6 +497,17 @@ If any of the related flags are also set, they override the mode's default behav
 		os.Setenv("TO_PASS", toPass)
 	}
 
+	rpmDBisOk := verifyRpmDB()
+
+	if *installPackagesPtr && !rpmDBisOk {
+		if t3cutil.StrToMode(*runModePtr) == t3cutil.ModeBadAss {
+			fatalLogStrs = append(fatalLogStrs, "RPM database check failed unable to install packages cannot continue in badass mode")
+		} else {
+			fatalLogStrs = append(fatalLogStrs, "RPM database check failed unable to install packages cannot continue")
+		}
+	}
+
+	toInfoLog = append(toInfoLog, fmt.Sprintf("rpm database is ok: %t", rpmDBisOk))
 	// set TSHome
 	var tsHome = ""
 	if *tsHomePtr != "" {
@@ -481,13 +518,13 @@ If any of the related flags are also set, they override the mode's default behav
 		tsHome = os.Getenv("TS_HOME") // check for the environment variable.
 		if tsHome != "" {
 			toInfoLog = append(toInfoLog, fmt.Sprintf("set TSHome from TS_HOME environment variable '%s'\n", TSHome))
-		} else { // finally check using the config file listing from the rpm package.
+		} else if rpmDBisOk { // check using the config file listing from the rpm package if rpmdb is ok.
 			tsHome = GetTSPackageHome()
 			if tsHome != "" {
 				toInfoLog = append(toInfoLog, fmt.Sprintf("set TSHome from the RPM config file  list '%s'\n", TSHome))
-			} else {
-				toInfoLog = append(toInfoLog, fmt.Sprintf("no override for TSHome was found, using the configured default: '%s'\n", TSHome))
 			}
+		} else if tsHome == "" {
+			toInfoLog = append(toInfoLog, fmt.Sprintf("no override for TSHome was found, using the configured default: '%s'\n", TSHome))
 		}
 	}
 
@@ -503,23 +540,23 @@ If any of the related flags are also set, they override the mode's default behav
 	if *useLocalATSVersionPtr {
 		atsVersionStr, err = GetATSVersionStr(tsHome)
 		if err != nil {
-			return Cfg{}, errors.New("getting local ATS version: " + err.Error())
+			fatalLogStrs = append(fatalLogStrs, "getting local ATS version: "+err.Error())
 		}
 	}
 	toInfoLog = append(toInfoLog, fmt.Sprintf("ATSVersionStr: '%s'\n", atsVersionStr))
 
 	usageStr := "basic usage: t3c-apply --traffic-ops-url=myurl --traffic-ops-user=myuser --traffic-ops-password=mypass --cache-host-name=my-cache"
 	if strings.TrimSpace(toURL) == "" {
-		return Cfg{}, errors.New("Missing required argument --traffic-ops-url or TO_URL environment variable. " + usageStr)
+		fatalLogStrs = append(fatalLogStrs, "Missing required argument --traffic-ops-url or TO_URL environment variable. "+usageStr)
 	}
 	if strings.TrimSpace(toUser) == "" {
-		return Cfg{}, errors.New("Missing required argument --traffic-ops-user or TO_USER environment variable. " + usageStr)
+		fatalLogStrs = append(fatalLogStrs, "Missing required argument --traffic-ops-user or TO_USER environment variable. "+usageStr)
 	}
 	if strings.TrimSpace(toPass) == "" {
-		return Cfg{}, errors.New("Missing required argument --traffic-ops-password or TO_PASS environment variable. " + usageStr)
+		fatalLogStrs = append(fatalLogStrs, "Missing required argument --traffic-ops-password or TO_PASS environment variable. "+usageStr)
 	}
 	if strings.TrimSpace(cacheHostName) == "" {
-		return Cfg{}, errors.New("Missing required argument --cache-host-name. " + usageStr)
+		fatalLogStrs = append(fatalLogStrs, "Missing required argument --cache-host-name. "+usageStr)
 	}
 
 	toURLParsed, err := url.Parse(toURL)
@@ -540,6 +577,7 @@ If any of the related flags are also set, they override the mode's default behav
 		CacheHostName:               cacheHostName,
 		SvcManagement:               svcManagement,
 		Retries:                     retries,
+		RpmDBOk:                     rpmDBisOk,
 		ReverseProxyDisable:         reverseProxyDisable,
 		SkipOSCheck:                 skipOsCheck,
 		UseStrategies:               useStrategies,
@@ -580,6 +618,13 @@ If any of the related flags are also set, they override the mode's default behav
 		return Cfg{}, errors.New("Initializing loggers: " + err.Error() + "\n")
 	}
 
+	if len(fatalLogStrs) > 0 {
+		for _, str := range fatalLogStrs {
+			str = strings.TrimSpace(str)
+			log.Errorln(str)
+		}
+		return Cfg{}, errors.New("fatal error has occurred")
+	}
 	for _, str := range modeLogStrs {
 		str = strings.TrimSpace(str)
 		if str == "" {

cache-config/t3c-apply/t3c-apply.go

@@ -22,7 +22,6 @@ package main
 import (
 	"encoding/json"
 	"errors"
-	"fmt"
 	"io/ioutil"
 	"os"
 	"path/filepath"
@@ -94,8 +93,8 @@ func Main() int {
 	var lock util.FileLock
 	cfg, err := config.GetCfg(Version, GitRevision)
 	if err != nil {
-		fmt.Println(err)
-		fmt.Println(FailureExitMsg)
+		log.Infoln(err)
+		log.Errorln(FailureExitMsg)
 		return ExitCodeConfigError
 	} else if cfg == (config.Cfg{}) { // user used the --help option
 		return ExitCodeSuccess
@@ -261,7 +260,7 @@ func Main() int {
 		// make sure we got the data necessary to check packages
 		log.Infoln("======== Didn't get all files, no package processing needed or possible ========")
 		metaData.InstalledPackages = oldMetaData.InstalledPackages
-	} else {
+	} else if cfg.RpmDBOk {
 		log.Infoln("======== Start processing packages  ========")
 		err = trops.ProcessPackages()
 		if err != nil {
@@ -276,6 +275,9 @@ func Main() int {
 			log.Errorf("Error verifying system services: %s\n", err.Error())
 			return GitCommitAndExit(ExitCodeServicesError, FailureExitMsg, cfg, metaData, oldMetaData)
 		}
+	} else {
+		log.Warnln("======== RPM DB checks failed, package processing not possible, using installed packages from  metadata if available========")
+		trops.ProcessPackagesWithMetaData(oldMetaData.InstalledPackages)
 	}
 
 	log.Debugf("Preparing to fetch the config files for %s, files: %s, syncdsUpdate: %s\n", cfg.CacheHostName, cfg.Files, syncdsUpdate)

cache-config/t3c-apply/torequest/torequest.go

@@ -587,10 +587,14 @@ func (r *TrafficOpsReq) CheckSystemServices() error {
 func (r *TrafficOpsReq) IsPackageInstalled(name string) bool {
 	for k, v := range r.Pkgs {
 		if strings.HasPrefix(k, name) {
+			log.Infof("Found in cache for '%s'", k)
 			return v
 		}
 	}
-
+	if !r.Cfg.RpmDBOk {
+		log.Warnf("RPM DB is corrupted cannot run IsPackageInstalled for '%s' and package metadata is unavailable", name)
+		return false
+	}
 	log.Infof("IsPackageInstalled '%v' not found in cache, querying rpm", name)
 	pkgArr, err := util.PackageInfo("pkg-query", name)
 	if err != nil {
@@ -1030,6 +1034,47 @@ func (r *TrafficOpsReq) ProcessPackages() error {
 	return nil
 }
 
+func pkgMetaDataToMap(pmd []string) map[string]bool {
+	pkgMap := map[string]bool{}
+	for _, pkg := range pmd {
+		pkgMap[pkg] = true
+	}
+	return pkgMap
+}
+
+func pkgMatch(pkgMetaData []string, pk string) bool {
+	for _, pkg := range pkgMetaData {
+		if strings.Contains(pk, pkg) {
+			return true
+		}
+	}
+	return false
+
+}
+
+// ProcessPackagesWithMetaData will attempt to get installed package data from
+// t3c-apply-metadata.json and log the results.
+func (r *TrafficOpsReq) ProcessPackagesWithMetaData(packageMetaData []string) error {
+	pkgs, err := getPackages(r.Cfg)
+	pkgMdataMap := pkgMetaDataToMap(packageMetaData)
+	if err != nil {
+		return fmt.Errorf("getting packages: %w", err)
+	}
+	for _, pkg := range pkgs {
+		fullPackage := pkg.Name + "-" + pkg.Version
+		if pkgMdataMap[fullPackage] {
+			log.Infof("package %s is assumed to be installed according to metadata file", fullPackage)
+			r.Pkgs[fullPackage] = true
+		} else if pkgMatch(packageMetaData, pkg.Name) {
+			log.Infof("package %s is assumed to be installed according to metadata, but doesn't match traffic ops pkg", fullPackage)
+			r.Pkgs[fullPackage] = true
+		} else {
+			log.Infof("package %s does not appear to be installed.", pkg.Name+"-"+pkg.Version)
+		}
+	}
+	return nil
+}
+
 func (r *TrafficOpsReq) RevalidateWhileSleeping(metaData *t3cutil.ApplyMetaData) (UpdateStatus, error) {
 	updateStatus, err := r.CheckRevalidateState(true)
 	if err != nil {