Skip to content

Commit

Permalink
Merge pull request #145 from aerospike/v7.1.0
Browse files Browse the repository at this point in the history 
V7.1.0
  • Loading branch information
@robertglonek
robertglonek authored Oct 5, 2023
2 parents bfab4cc + ac9392a commit e3bb6ff
Show file tree
Hide file tree
Showing 16 changed files with 8,726 additions and 83,017 deletions.
2 changes: 1 addition & 1 deletion src/cmdAgiExecProxy.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -265,7 +265,7 @@ func (c *agiExecProxyCmd) handleList(w http.ResponseWriter, r *http.Request) {
}
w.WriteHeader(http.StatusOK)
out := []byte(`<html><head><title>AGI URLs</title></head><body><center>
<a href="/" target="_blank"><h1>Grafana</h1></a>
<a href="/d/dashList/dashboard-list?from=now-7d&to=now&var-MaxIntervalSeconds=30&var-ProduceDelta&var-ClusterName=All&var-NodeIdent=All&var-Namespace=All&var-Histogram=NONE&var-HistogramDev=NONE&var-HistogramUs=NONE&var-HistogramCount=NONE&var-HistogramSize=NONE&var-XdrDcName=All&var-xdr5dc=All&var-warnC=All&var-warnCtx=All&var-errC=All&var-errCtx=All&orgId=1" target="_blank"><h1>Grafana</h1></a>
<a href="/agi/ttyd" target="_blank"><h1>Web Console (ttyd)</h1></a>
<a href="/agi/filebrowser" target="_blank"><h1>File Browser</h1></a>
<a href="/agi/reingest" target="_blank"><h1>Retrigger Ingest</h1></a>
Expand Down
68 changes: 56 additions & 12 deletions src/cmdClusterPartitionConf.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ type clusterPartitionConfCmd struct {
FilterPartitions TypeFilterRange `short:"p" long:"filter-partitions" description:"Select partitions on each disk by number; 0=use entire disk itself, ex: 1,2,4-8" default:"ALL"`
FilterType string `short:"t" long:"filter-type" description:"what disk types to select, options: nvme/local or ebs/persistent" default:"ALL"`
Namespace string `short:"m" long:"namespace" description:"namespace to modify the settings for" default:""`
ConfDest string `short:"o" long:"configure" description:"what to configure the selections as; options: device|shadow|pi-flash|si-flash" default:""`
ConfDest string `short:"o" long:"configure" description:"what to configure the selections as; options: memory|device|shadow|pi-flash|si-flash" default:""`
MountsSizeLimitPct float64 `short:"s" long:"mounts-size-limit-pct" description:"specify %% for what the mounts-size-limit should be set to for flash configs" default:"90"`
parallelThreadsLongCmd
Help helpCmd `command:"help" subcommands-optional:"true" description:"Print help"`
Expand All @@ -40,8 +40,8 @@ func (c *clusterPartitionConfCmd) Execute(args []string) error {
} else if c.FilterType == "persistent" {
c.FilterType = "ebs"
}
if !inslice.HasString([]string{"device", "shadow", "pi-flash", "si-flash", "allflash"}, c.ConfDest) {
return fmt.Errorf("configure options must be one of: device, shadow, pi-flash, si-flash, allflash")
if !inslice.HasString([]string{"memory", "device", "shadow", "pi-flash", "si-flash", "allflash"}, c.ConfDest) {
return fmt.Errorf("configure options must be one of: memory, device, shadow, pi-flash, si-flash, allflash")
}
if c.Namespace == "" {
return fmt.Errorf("namespace name is required")
Expand Down Expand Up @@ -110,7 +110,22 @@ func (c *clusterPartitionConfCmd) doParallel(nodeNo int, disks map[int]map[int]b
}

func (c *clusterPartitionConfCmd) do(nodeNo int, disks map[int]map[int]blockDevices, filterDiskCount int, filterPartCount int) error {
outn, err := b.RunCommands(c.ClusterName.String(), [][]string{{"cat", "/etc/aerospike/aerospike.conf"}}, []int{nodeNo})
is7 := "mounts-budget"
outn, err := b.RunCommands(c.ClusterName.String(), [][]string{{"cat", "/opt/aerolab.aerospike.version"}}, []int{nodeNo})
if err != nil {
log.Printf("could not read /opt/aerolab.aerospike.version on node %d, assuming asd version 7: %s", nodeNo, err)
} else {
major := strings.Trim(strings.Split(string(outn[0]), ".")[0], "\r\n\t ")
majorInt, err := strconv.Atoi(major)
if err != nil {
log.Printf("could not parse /opt/aerolab.aerospike.version on node %d, assuming asd version 7: %s", nodeNo, err)
} else {
if majorInt < 7 {
is7 = "mounts-size-limit"
}
}
}
outn, err = b.RunCommands(c.ClusterName.String(), [][]string{{"cat", "/etc/aerospike/aerospike.conf"}}, []int{nodeNo})
if err != nil {
return fmt.Errorf("could not read aerospike.conf on node %d: %s", nodeNo, err)
}
Expand All @@ -123,6 +138,19 @@ func (c *clusterPartitionConfCmd) do(nodeNo int, disks map[int]map[int]blockDevi
cc.NewStanza("namespace " + c.Namespace)
}
switch c.ConfDest {
case "memory":
for _, key := range cc.Stanza("namespace " + c.Namespace).ListKeys() {
if strings.HasPrefix(key, "storage-engine") && (!strings.HasSuffix(key, "memory") || cc.Stanza("namespace "+c.Namespace).Type("storage-engine memory") != aeroconf.ValueStanza) {
cc.Stanza("namespace " + c.Namespace).Delete(key)
}
}
if cc.Stanza("namespace "+c.Namespace).Type("storage-engine memory") == aeroconf.ValueNil {
cc.Stanza("namespace " + c.Namespace).NewStanza("storage-engine memory")
}
cc.Stanza("namespace " + c.Namespace).Stanza("storage-engine memory").Delete("device")
cc.Stanza("namespace " + c.Namespace).Stanza("storage-engine memory").Delete("file")
cc.Stanza("namespace " + c.Namespace).Stanza("storage-engine memory").Delete("filesize")
cc.Stanza("namespace " + c.Namespace).Stanza("storage-engine memory").Delete("data-size")
case "device":
for _, key := range cc.Stanza("namespace " + c.Namespace).ListKeys() {
if strings.HasPrefix(key, "storage-engine") && (!strings.HasSuffix(key, "device") || cc.Stanza("namespace "+c.Namespace).Type("storage-engine device") != aeroconf.ValueStanza) {
Expand All @@ -136,10 +164,10 @@ func (c *clusterPartitionConfCmd) do(nodeNo int, disks map[int]map[int]blockDevi
cc.Stanza("namespace " + c.Namespace).Stanza("storage-engine device").Delete("file")
cc.Stanza("namespace " + c.Namespace).Stanza("storage-engine device").Delete("filesize")
case "shadow":
if cc.Stanza("namespace "+c.Namespace).Type("storage-engine device") != aeroconf.ValueStanza {
return fmt.Errorf("storage-engine device configuration not found for namespace")
if cc.Stanza("namespace "+c.Namespace).Type("storage-engine device") != aeroconf.ValueStanza && cc.Stanza("namespace "+c.Namespace).Type("storage-engine memory") != aeroconf.ValueStanza {
return fmt.Errorf("storage-engine device|memory configuration not found for namespace")
}
if !inslice.HasString(cc.Stanza("namespace "+c.Namespace).Stanza("storage-engine device").ListKeys(), "device") {
if !inslice.HasString(cc.Stanza("namespace "+c.Namespace).Stanza("storage-engine device").ListKeys(), "device") && !inslice.HasString(cc.Stanza("namespace "+c.Namespace).Stanza("storage-engine memory").ListKeys(), "device") {
return fmt.Errorf("no device configured for given namespace, cannot add shadow devices")
}
case "allflash":
Expand Down Expand Up @@ -205,6 +233,14 @@ func (c *clusterPartitionConfCmd) do(nodeNo int, disks map[int]map[int]blockDevi
})
for _, p := range useParts {
switch c.ConfDest {
case "memory":
vals, err := cc.Stanza("namespace " + c.Namespace).Stanza("storage-engine memory").GetValues("device")
if err != nil {
return err
}
pp := p.Path
vals = append(vals, &pp)
cc.Stanza("namespace "+c.Namespace).Stanza("storage-engine memory").SetValues("device", vals)
case "device":
vals, err := cc.Stanza("namespace " + c.Namespace).Stanza("storage-engine device").GetValues("device")
if err != nil {
Expand All @@ -214,7 +250,11 @@ func (c *clusterPartitionConfCmd) do(nodeNo int, disks map[int]map[int]blockDevi
vals = append(vals, &pp)
cc.Stanza("namespace "+c.Namespace).Stanza("storage-engine device").SetValues("device", vals)
case "shadow":
vals, err := cc.Stanza("namespace " + c.Namespace).Stanza("storage-engine device").GetValues("device")
ntype := "memory"
if cc.Stanza("namespace "+c.Namespace).Stanza("storage-engine device") != nil {
ntype = "device"
}
vals, err := cc.Stanza("namespace " + c.Namespace).Stanza("storage-engine " + ntype).GetValues("device")
if err != nil {
return err
}
Expand All @@ -231,7 +271,7 @@ func (c *clusterPartitionConfCmd) do(nodeNo int, disks map[int]map[int]blockDevi
if !found {
return errors.New("not enough primary devices for chosen shadow devices")
}
cc.Stanza("namespace "+c.Namespace).Stanza("storage-engine device").SetValues("device", vals)
cc.Stanza("namespace "+c.Namespace).Stanza("storage-engine "+ntype).SetValues("device", vals)
case "allflash":
fallthrough
case "pi-flash":
Expand Down Expand Up @@ -260,7 +300,7 @@ func (c *clusterPartitionConfCmd) do(nodeNo int, disks map[int]map[int]blockDevi
}
fsSize = strconv.Itoa(int(float64(fsSizeI)*c.MountsSizeLimitPct/100)) + suffix
}
cc.Stanza("namespace "+c.Namespace).Stanza("index-type flash").SetValue("mounts-size-limit", fsSize)
cc.Stanza("namespace "+c.Namespace).Stanza("index-type flash").SetValue(is7, fsSize)
case "si-flash":
vals, err := cc.Stanza("namespace " + c.Namespace).Stanza("sindex-type flash").GetValues("mount")
if err != nil {
Expand All @@ -287,11 +327,15 @@ func (c *clusterPartitionConfCmd) do(nodeNo int, disks map[int]map[int]blockDevi
}
fsSize = strconv.Itoa(int(float64(fsSizeI)*c.MountsSizeLimitPct/100)) + suffix
}
cc.Stanza("namespace "+c.Namespace).Stanza("sindex-type flash").SetValue("mounts-size-limit", fsSize)
cc.Stanza("namespace "+c.Namespace).Stanza("sindex-type flash").SetValue(is7, fsSize)
}
}
if c.ConfDest == "shadow" {
vals, err := cc.Stanza("namespace " + c.Namespace).Stanza("storage-engine device").GetValues("device")
ntype := "memory"
if cc.Stanza("namespace "+c.Namespace).Stanza("storage-engine device") != nil {
ntype = "device"
}
vals, err := cc.Stanza("namespace " + c.Namespace).Stanza("storage-engine " + ntype).GetValues("device")
if err != nil {
return err
}
Expand Down
13 changes: 11 additions & 2 deletions src/cmdConfGenerator.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,20 +2,29 @@ package main

import (
"github.com/aerospike/aerolab/confeditor"
"github.com/aerospike/aerolab/confeditor7"
flags "github.com/rglonek/jeddevdk-goflags"
)

type confGeneratorCmd struct {
Path flags.Filename `short:"f" long:"filename" description:"file name to read/write/generate" default:"aerospike.conf"`
DisableColors bool `short:"c" long:"--no-colors" description:"set to operate in no-color mode"`
Pre7 bool `short:"6" long:"pre-7" description:"set to to generator for pre-version-7 aerospike"`
DisableColors bool `short:"c" long:"no-colors" description:"set to operate in no-color mode"`
Help helpCmd `command:"help" subcommands-optional:"true" description:"Print help"`
}

func (c *confGeneratorCmd) Execute(args []string) error {
if earlyProcessV2(args, false) {
return nil
}
e := &confeditor.Editor{
if c.Pre7 {
e := &confeditor.Editor{
Path: string(c.Path),
Colors: !c.DisableColors,
}
return e.Run()
}
e := &confeditor7.Editor{
Path: string(c.Path),
Colors: !c.DisableColors,
}
Expand Down
50 changes: 48 additions & 2 deletions src/cmdInventoryInstanceTypes.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,9 @@ import (
"encoding/json"
"errors"
"fmt"
"log"
"os"
"os/exec"
"sort"
"strconv"
"strings"
Expand All @@ -17,6 +19,7 @@ import (
)

type inventoryInstanceTypesCmd struct {
NoPaginate bool `long:"no-paginate" description:"set to disable vertical and horizontal pagination"`
Json bool `short:"j" long:"json" description:"Provide output in json format"`
JsonPretty bool `short:"p" long:"pretty" description:"Provide json output with line-feeds and indentations"`
Arm bool `short:"a" long:"arm" description:"Set to look for ARM instances instead of amd64"`
Expand Down Expand Up @@ -142,6 +145,9 @@ func (c *inventoryInstanceTypesCmd) Execute(args []string) error {
if earlyProcess(args) {
return nil
}
if c.JsonPretty {
c.Json = true
}
if a.opts.Config.Backend.Type == "docker" {
return errors.New("feature not available on docker")
}
Expand Down Expand Up @@ -172,10 +178,16 @@ func (c *inventoryInstanceTypesCmd) Execute(args []string) error {
return err
}

isColor := true
if _, ok := os.LookupEnv("NO_COLOR"); ok || os.Getenv("CLICOLOR") == "0" {
isColor = false
}
pipeLess := !c.NoPaginate

t := table.NewWriter()
// For now, don't set the allowed row lenght, wrapping is better
// until we do something more clever...
if isatty.IsTerminal(os.Stdout.Fd()) {
if isatty.IsTerminal(os.Stdout.Fd()) && isColor {
// fmt.Println("Is Terminal")
t.SetStyle(table.StyleColoredBlackOnCyanWhite)

Expand All @@ -184,7 +196,7 @@ func (c *inventoryInstanceTypesCmd) Execute(args []string) error {
fmt.Println("Couldn't get terminal width")
}
// t.SetAllowedRowLength(s.Width)
} else if isatty.IsCygwinTerminal(os.Stdout.Fd()) {
} else if isatty.IsCygwinTerminal(os.Stdout.Fd()) && isColor {
// fmt.Println("Is Cygwin/MSYS2 Terminal")
t.SetStyle(table.StyleColoredBlackOnCyanWhite)

Expand All @@ -194,10 +206,44 @@ func (c *inventoryInstanceTypesCmd) Execute(args []string) error {
}
// t.SetAllowedRowLength(s.Width)
} else {
pipeLess = false
fmt.Fprintln(os.Stderr, "aerolab does not have a stable CLI interface. Use with caution in scripts.\nIn scripts, the JSON output should be used for stability.")
t.SetStyle(table.StyleDefault)
}

lessCmd := ""
lessParams := []string{}
if pipeLess {
lessCmd, lessParams = getPaginationCommand()
}

if lessCmd != "" {
origStdout := os.Stdout // store original
origStderr := os.Stderr // store original
defer func() { // on exit, last thing we do, we recover stdout/stderr
os.Stdout = origStdout
os.Stderr = origStderr
}()
less := exec.Command(lessCmd, lessParams...)
less.Stdout = origStdout // less will write
less.Stderr = origStderr // less will write
r, w, err := os.Pipe() // writer writes, reader reads
if err == nil {
os.Stdout = w // we will write to os.Pipe
os.Stderr = w // we will write to os.Pipe
less.Stdin = r // less will read from os.Pipe
err = less.Start() // start less so it can do it's magic
if err != nil { // on pagination failure, revert to non-paginated output
os.Stdout = origStdout
os.Stderr = origStderr
log.Printf("Pagination failed, %s returned: %s", lessCmd, err)
} else {
defer less.Wait() // after closing w, we should wait for less to finish before exiting
defer w.Close() // must close or less will wait for more input
}
}
}

//t.SetTitle("INSTANCES")
t.AppendHeader(table.Row{"Instance Name", "CPUs", "Ram GB", "Local Disks", "Local Disk Total Size GB", "On-Demand $/hour", "On-Demand $/day", "On-Demand $/month"})
for _, v := range instanceTypes {
Expand Down
Loading

0 comments on commit e3bb6ff

Please sign in to comment.