Merge pull request #336 from bmc-toolbox/firmwaresmc

Supermicro X11 BIOS, BMC firmware install support

errors/errors.go

@@ -95,6 +95,16 @@ var (
 
 	// ErrIncompatibleProvider is returned by Open() when the device is not compatible with the provider
 	ErrIncompatibleProvider = errors.New("provider not compatible with device")
+
+	// ErrBMCColdResetRequired is returned when a BMC cold reset is required.
+	ErrBMCColdResetRequired = errors.New("BMC cold reset required")
+
+	// ErrHostPowercycleRequired is returned when a host powercycle is required.
+	ErrHostPowercycleRequired = errors.New("Host power cycle required")
+
+	// ErrSessionExpired is returned when the BMC session is not valid
+	// the receiver can then choose to request a new session.
+	ErrSessionExpired = errors.New("session expired")
 )
 
 type ErrUnsupportedHardware struct {

examples/install-firmware/doc.go

@@ -1,24 +1,58 @@
 /*
-install-firmware is an example commmand that utilizes the 'v1' bmclib interface
+install-firmware is an example command that utilizes the 'v1' bmclib interface
 methods to flash a firmware image to a BMC.
 
-	$ go run ./examples/v1/install-firmware/main.go -h
-	Usage of /tmp/go-build2950657412/b001/exe/main:
-		-cert-pool string
-					Path to an file containing x509 CAs. An empty string uses the system CAs. Only takes effect when --secure-tls=true
-		-firmware string
-					The local path of the firmware to install
-		-host string
-					BMC hostname to connect to
-		-password string
-					Username to login with
-		-port int
-					BMC port to connect to (default 443)
-		-secure-tls
-					Enable secure TLS
-		-user string
-					Username to login with
-		-version string
-					The firmware version being installed
+Note: The example installs the firmware and polls until the status until the install is complete,
+and if required by the install process - power cycles the host.
+
+							$ go run ./examples/v1/install-firmware/main.go -h
+							Usage of /tmp/go-build2950657412/b001/exe/main:
+								-cert-pool string
+											Path to an file containing x509 CAs. An empty string uses the system CAs. Only takes effect when -secure-tls=true
+								-firmware string
+											The local path of the firmware to install
+								-host string
+											BMC hostname to connect to
+								-password string
+											Username to login with
+								-port int
+											BMC port to connect to (default 443)
+								-secure-tls
+											Enable secure TLS
+								-user string
+											Username to login with
+								-version string
+											The firmware version being installed
+
+						   # install bios firmware on a supermicro X11
+						   #
+						   $ go run .  -host 192.168.1.1 -user ADMIN -password hunter2 -component bios -firmware BIOS_X11DPH-0981_20220208_3.6_STD.bin
+						   INFO[0007] set firmware install mode                     component=BIOS ip="https://192.168.1.1" model=X11DPH-T
+						   INFO[0011] uploading firmware                            component=BIOS ip="https://192.168.1.1" model=X11DPH-T
+						   INFO[0091] verifying uploaded firmware                   component=BIOS ip="https://192.168.1.1" model=X11DPH-T
+						   INFO[0105] initiating firmware install                   component=BIOS ip="https://192.168.1.1" model=X11DPH-T
+						   INFO[0115] firmware install running                      component=bios state=running
+						   INFO[0132] firmware install running                      component=bios state=running
+						   ...
+						   ...
+	                       INFO[0628] firmware install running                      component=bios state=running
+	                       INFO[0635] host powercycle required                      component=bios state=powercycle-host
+	                       INFO[0637] host power cycled, all done!                  component=bios state=powercycle-host
+
+
+
+							# install bmc firmware on a supermicro X11
+							#
+							$ go run .  -host 192.168.1.1 -user ADMIN -password hunter2 -component bmc -firmware BMC_X11AST2500-4101MS_20220225_01.74.02_STD.bin
+			                INFO[0007] setting device to firmware install mode       component=BMC ip="https://192.168.1.1"
+			                INFO[0009] uploading firmware                            ip="https://192.168.1.1"
+			                INFO[0045] verifying uploaded firmware                   ip="https://192.168.1.1"
+			                INFO[0047] initiating firmware install                   ip="https://192.168.1.1"
+			                INFO[0079] firmware install running                      component=bmc state=running
+			                INFO[0085] firmware install running                      component=bmc state=running
+			                ...
+							...
+							INFO[0233] firmware install running                      component=bmc state=running
+		                    INFO[0238] firmware install completed                    component=bmc state=complete
 */
 package main

examples/install-firmware/main.go

@@ -3,15 +3,17 @@ package main
 import (
 	"context"
 	"crypto/x509"
+	"errors"
 	"flag"
 	"io/ioutil"
 	"log"
 	"os"
+	"strings"
 	"time"
 
 	bmclib "github.com/bmc-toolbox/bmclib/v2"
 	"github.com/bmc-toolbox/bmclib/v2/constants"
-	"github.com/bmc-toolbox/common"
+	bmclibErrs "github.com/bmc-toolbox/bmclib/v2/errors"
 	"github.com/bombsimon/logrusr/v2"
 	"github.com/sirupsen/logrus"
 )
@@ -20,24 +22,33 @@ func main() {
 	user := flag.String("user", "", "Username to login with")
 	pass := flag.String("password", "", "Username to login with")
 	host := flag.String("host", "", "BMC hostname to connect to")
+	component := flag.String("component", "", "Component to be updated (bmc, bios.. etc)")
 	withSecureTLS := flag.Bool("secure-tls", false, "Enable secure TLS")
 	certPoolPath := flag.String("cert-pool", "", "Path to an file containing x509 CAs. An empty string uses the system CAs. Only takes effect when --secure-tls=true")
 	firmwarePath := flag.String("firmware", "", "The local path of the firmware to install")
 	firmwareVersion := flag.String("version", "", "The firmware version being installed")
 
 	flag.Parse()
 
-	ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
+	ctx, cancel := context.WithTimeout(context.Background(), 60*time.Minute)
 	defer cancel()
 
 	l := logrus.New()
-	l.Level = logrus.DebugLevel
+	l.Level = logrus.TraceLevel
 	logger := logrusr.New(l)
 
 	if *host == "" || *user == "" || *pass == "" {
 		l.Fatal("required host/user/pass parameters not defined")
 	}
-	clientOpts := []bmclib.Option{bmclib.WithLogger(logger)}
+
+	if *component == "" {
+		l.Fatal("component parameter required (must be a component slug - bmc, bios etc)")
+	}
+
+	clientOpts := []bmclib.Option{
+		bmclib.WithLogger(logger),
+		bmclib.WithPerProviderTimeout(time.Minute * 30),
+	}
 
 	if *withSecureTLS {
 		var pool *x509.CertPool
@@ -61,32 +72,73 @@ func main() {
 
 	defer cl.Close(ctx)
 
-	// collect inventory
-	inventory, err := cl.Inventory(ctx)
-	if err != nil {
-		l.Fatal(err)
-	}
-
-	l.WithField("bmc-version", inventory.BMC.Firmware.Installed).Info()
-
 	// open file handle
 	fh, err := os.Open(*firmwarePath)
 	if err != nil {
 		l.Fatal(err)
 	}
 	defer fh.Close()
 
-	// SlugBMC hardcoded here, this can be any of the existing component slugs from devices/constants.go
-	// assuming that the BMC provider implements the required component firmware update support
-	taskID, err := cl.FirmwareInstall(ctx, common.SlugBMC, constants.FirmwareApplyOnReset, true, fh)
+	taskID, err := cl.FirmwareInstall(ctx, *component, constants.FirmwareApplyOnReset, true, fh)
 	if err != nil {
-		l.Error(err)
+		l.Fatal(err)
 	}
 
-	state, err := cl.FirmwareInstallStatus(ctx, taskID, common.SlugBMC, *firmwareVersion)
-	if err != nil {
-		log.Fatal(err)
-	}
+	for {
+		if ctx.Err() != nil {
+			l.Fatal(ctx.Err())
+		}
+
+		state, err := cl.FirmwareInstallStatus(ctx, *firmwareVersion, *component, taskID)
+		if err != nil {
+			// when its under update a connection refused is returned
+			if strings.Contains(err.Error(), "connection refused") || strings.Contains(err.Error(), "operation timed out") {
+				l.Info("BMC refused connection, BMC most likely resetting...")
+				time.Sleep(2 * time.Second)
+
+				continue
+			}
 
-	l.WithField("state", state).Info("BMC firmware install state")
+			if errors.Is(err, bmclibErrs.ErrSessionExpired) || strings.Contains(err.Error(), "session expired") {
+				err := cl.Open(ctx)
+				if err != nil {
+					l.Fatal(err, "bmc re-login failed")
+				}
+
+				l.WithFields(logrus.Fields{"state": state, "component": *component}).Info("BMC session expired, logging in...")
+
+				continue
+			}
+
+			log.Fatal(err)
+		}
+
+		switch state {
+		case constants.FirmwareInstallRunning:
+			l.WithFields(logrus.Fields{"state": state, "component": *component}).Info("firmware install running")
+
+		case constants.FirmwareInstallFailed:
+			l.WithFields(logrus.Fields{"state": state, "component": *component}).Info("firmware install failed")
+			os.Exit(1)
+
+		case constants.FirmwareInstallComplete:
+			l.WithFields(logrus.Fields{"state": state, "component": *component}).Info("firmware install completed")
+			os.Exit(0)
+
+		case constants.FirmwareInstallPowerCyleHost:
+			l.WithFields(logrus.Fields{"state": state, "component": *component}).Info("host powercycle required")
+
+			if _, err := cl.SetPowerState(ctx, "cycle"); err != nil {
+				l.WithFields(logrus.Fields{"state": state, "component": *component}).Info("error power cycling host for install")
+				os.Exit(1)
+			}
+
+			l.WithFields(logrus.Fields{"state": state, "component": *component}).Info("host power cycled, all done!")
+			os.Exit(0)
+		default:
+			l.WithFields(logrus.Fields{"state": state, "component": *component}).Info("unknown state returned")
+		}
+
+		time.Sleep(2 * time.Second)
+	}
 }

internal/ipmi/ipmi.go

@@ -90,7 +90,7 @@ func (i *Ipmi) run(ctx context.Context, command []string) (output string, err er
 	}
 	for _, cipherString := range ipmiCiphers {
 		ipmiCmd := append(ipmiArgs, "-C", cipherString)
-		i.log.V(1).Info("ipmitool options", "opts", formatOptions(ipmiCmd))
+		i.log.V(3).Info("ipmitool options", "opts", formatOptions(ipmiCmd))
 		ipmiCmd = append(ipmiCmd, command...)
 		cmd := exec.CommandContext(ctx, i.ipmitool, ipmiCmd...)
 		cmd.Env = []string{fmt.Sprintf("IPMITOOL_PASSWORD=%s", i.Password)}

providers/supermicro/errors.go

@@ -0,0 +1,35 @@
+package supermicro
+
+import (
+	"fmt"
+	"strconv"
+
+	"github.com/pkg/errors"
+)
+
+var (
+	ErrQueryFRUInfo = errors.New("FRU information query returned error")
+)
+
+type UnexpectedResponseError struct {
+	payload    string
+	response   string
+	statusCode string
+}
+
+func (e *UnexpectedResponseError) Error() string {
+	return fmt.Sprintf(
+		"unexpected response - statusCode: %s, payload: %s, response: %s",
+		e.statusCode,
+		e.payload,
+		e.response,
+	)
+}
+
+func unexpectedResponseErr(payload, response []byte, statusCode int) error {
+	return &UnexpectedResponseError{
+		string(payload),
+		string(response),
+		strconv.Itoa(statusCode),
+	}
+}

providers/supermicro/firmware.go

@@ -0,0 +1,101 @@
+package supermicro
+
+import (
+	"context"
+	"io"
+	"os"
+	"strings"
+	"time"
+
+	bmclibErrs "github.com/bmc-toolbox/bmclib/v2/errors"
+	"github.com/bmc-toolbox/common"
+	"github.com/pkg/errors"
+)
+
+// FirmwareInstall uploads and initiates firmware update for the component
+func (c *Client) FirmwareInstall(ctx context.Context, component, applyAt string, forceInstall bool, reader io.Reader) (jobID string, err error) {
+	if err := c.deviceSupported(ctx); err != nil {
+		return "", errors.Wrap(bmclibErrs.ErrFirmwareInstall, err.Error())
+	}
+
+	var size int64
+	if file, ok := reader.(*os.File); ok {
+		finfo, err := file.Stat()
+		if err != nil {
+			c.log.V(2).Error(err, "unable to determine file size")
+		}
+
+		size = finfo.Size()
+	}
+
+	// expect atleast 30 minutes left in the deadline to proceed with the update
+	d, _ := ctx.Deadline()
+	if time.Until(d) < 30*time.Minute {
+		return "", errors.New("remaining context deadline insufficient to perform update: " + time.Until(d).String())
+	}
+
+	component = strings.ToUpper(component)
+
+	switch component {
+	case common.SlugBIOS:
+		err = c.firmwareInstallBIOS(ctx, reader, size)
+	case common.SlugBMC:
+		err = c.firmwareInstallBMC(ctx, reader, size)
+	default:
+		return "", errors.Wrap(bmclibErrs.ErrFirmwareInstall, "component unsupported: "+component)
+	}
+
+	if err != nil {
+		err = errors.Wrap(bmclibErrs.ErrFirmwareInstall, err.Error())
+	}
+
+	return jobID, err
+}
+
+// FirmwareInstallStatus returns the status of the firmware install process
+func (c *Client) FirmwareInstallStatus(ctx context.Context, installVersion, component, taskID string) (string, error) {
+	component = strings.ToUpper(component)
+
+	switch component {
+	case common.SlugBMC:
+		return c.statusBMCFirmwareInstall(ctx)
+	case common.SlugBIOS:
+		return c.statusBIOSFirmwareInstall(ctx)
+	default:
+		return "", errors.Wrap(bmclibErrs.ErrFirmwareInstallStatus, "component unsupported: "+component)
+	}
+}
+
+func (c *Client) deviceSupported(ctx context.Context) error {
+	errBoardPartNumUnknown := errors.New("baseboard part number unknown")
+	errBoardUnsupported := errors.New("feature not supported/implemented for device")
+
+	// Its likely this works on all X11's
+	// for now, we list only the ones its been tested on.
+	//
+	// board part numbers
+	//
+	supported := []string{
+		"X11SCM-F",
+		"X11DPH-T",
+	}
+
+	data, err := c.fruInfo(ctx)
+	if err != nil {
+		return err
+	}
+
+	if data.Board == nil || strings.TrimSpace(data.Board.PartNum) == "" {
+		return errors.Wrap(errBoardPartNumUnknown, "baseboard part number empty")
+	}
+
+	c.model = strings.TrimSpace(data.Board.PartNum)
+
+	for _, b := range supported {
+		if strings.EqualFold(b, strings.TrimSpace(data.Board.PartNum)) {
+			return nil
+		}
+	}
+
+	return errors.Wrap(errBoardUnsupported, data.Board.PartNum)
+}