diff --git a/pkg/invoice/invoice_golden_test.go b/pkg/invoice/invoice_golden_test.go index eb73f6f..255046b 100644 --- a/pkg/invoice/invoice_golden_test.go +++ b/pkg/invoice/invoice_golden_test.go @@ -64,8 +64,8 @@ func (q fakeQuerier) Query(ctx context.Context, query string, ts time.Time, _ .. res = append(res, &model.Sample{ Metric: map[model.LabelName]model.LabelValue{ "product": model.LabelValue(k), - "category": model.LabelValue(fmt.Sprintf("%s:%s", sk.Zone, sk.Namespace)), - "tenant": model.LabelValue(sk.Tenant), + "category": model.LabelValue(fmt.Sprintf("%s:%s", sk.Part(1), sk.Part(3))), + "tenant": model.LabelValue(sk.Part(2)), }, Value: s.Value, }) diff --git a/pkg/report/report.go b/pkg/report/report.go index a706e44..5f62312 100644 --- a/pkg/report/report.go +++ b/pkg/report/report.go @@ -118,10 +118,10 @@ func processSample(ctx context.Context, tx *sqlx.Tx, ts time.Time, query db.Quer var upsertedTenant db.Tenant err = upsertTenant(ctx, tx, &upsertedTenant, db.Tenant{ - Source: skey.Tenant, + Source: skey.Tenant(), }, ts) if err != nil { - return fmt.Errorf("failed to upsert tenant '%s': %w", skey.Tenant, err) + return fmt.Errorf("failed to upsert tenant '%s': %w", skey.Tenant(), err) } var upsertedCategory db.Category diff --git a/pkg/sourcekey/sourcekey.go b/pkg/sourcekey/sourcekey.go index 73bc343..f599a1a 100644 --- a/pkg/sourcekey/sourcekey.go +++ b/pkg/sourcekey/sourcekey.go @@ -2,6 +2,9 @@ package sourcekey import ( "fmt" + "math" + "math/bits" + "sort" "strings" ) @@ -10,119 +13,97 @@ const elementSeparator = ":" // SourceKey represents a source key to look up dimensions objects (currently queries and products). // It implements the lookup logic found in https://kb.vshn.ch/appuio-cloud/references/architecture/metering-data-flow.html#_system_idea. type SourceKey struct { - Query string - Zone string - Tenant string - Namespace string - - Class string + parts []string } // Parse parses a source key in the format of "query:zone:tenant:namespace:class" or "query:zone:tenant:namespace". func Parse(raw string) (SourceKey, error) { parts := strings.Split(raw, elementSeparator) - if len(parts) == 4 { - return SourceKey{parts[0], parts[1], parts[2], parts[3], ""}, nil - } else if len(parts) == 5 { - return SourceKey{parts[0], parts[1], parts[2], parts[3], parts[4]}, nil + if parts[len(parts)-1] == "" { + parts = parts[0 : len(parts)-1] + } + if len(parts) >= 4 { + return SourceKey{parts}, nil } - return SourceKey{}, fmt.Errorf("expected key with 4 to 5 elements separated by `%s` got %d", elementSeparator, len(parts)) + return SourceKey{}, fmt.Errorf("expected key with at least 4 elements separated by `%s` got %d", elementSeparator, len(parts)) +} + +// Tenant returns the third element of the source key which was historically used as the tenant. +// +// Deprecated: We would like to get rid of this and read the tenant from a metric label. +func (k SourceKey) Tenant() string { + return k.parts[2] +} + +// Part returns the i-th part of the source key, or an empty string if no such part exists +func (k SourceKey) Part(i int) string { + if i < len(k.parts) { + return k.parts[i] + } + return "" } // String returns the string representation "query:zone:tenant:namespace:class" of the key. func (k SourceKey) String() string { - elements := []string{k.Query, k.Zone, k.Tenant, k.Namespace} - if k.Class != "" { - elements = append(elements, k.Class) - } - return strings.Join(elements, elementSeparator) + return strings.Join(k.parts, elementSeparator) } // LookupKeys generates lookup keys for a dimension object in the database. // The logic is described here: https://kb.vshn.ch/appuio-cloud/references/architecture/metering-data-flow.html#_system_idea func (k SourceKey) LookupKeys() []string { - return generateSourceKeys(k.Query, k.Zone, k.Tenant, k.Namespace, k.Class) -} -func generateSourceKeys(query, zone, tenant, namespace, class string) []string { keys := make([]string, 0) - base := []string{query, zone, tenant, namespace} - wildcardPositions := []int{1, 2} - - if class != "" { - wildcardPositions = append(wildcardPositions, 3) - base = append(base, class) - } - - for i := len(base); i > 0; i-- { - keys = append(keys, strings.Join(base[:i], elementSeparator)) - - for j := 1; j < len(wildcardPositions)+1; j++ { - perms := combinations(wildcardPositions, j) - for _, wcpos := range reverse(perms) { - elements := append([]string{}, base[:i]...) - for _, p := range wcpos { - elements[p] = "*" + currentKeyBase := k.parts + + for len(currentKeyBase) > 1 { + // For the base key of a given length l, the inner l-2 elements are to be replaced with wildcards in all possible combinations. + // To that end, generate 2^(l-2) binary numbers, sort them by specificity, and then for each number generate a key where + // for each 1-digit, the element is replaced with a wildcard (and for a 0-digit, the element is kept as-is). + innerLength := len(currentKeyBase) - 2 + nums := makeRange(0, int(math.Pow(2, float64(innerLength)))) + sort.Sort(sortBySpecificity(nums)) + for i := range nums { + currentKeyElements := make([]string, 0) + currentKeyElements = append(currentKeyElements, currentKeyBase[0]) + for digit := 0; digit < innerLength; digit++ { + if nums[i]&uint(math.Pow(2, float64(innerLength-1-digit))) > 0 { + currentKeyElements = append(currentKeyElements, "*") + } else { + currentKeyElements = append(currentKeyElements, currentKeyBase[1+digit]) } - keys = append(keys, strings.Join(elements, elementSeparator)) } + currentKeyElements = append(currentKeyElements, currentKeyBase[len(currentKeyBase)-1]) + keys = append(keys, strings.Join(currentKeyElements, elementSeparator)) } - if i > 2 { - wildcardPositions = wildcardPositions[:len(wildcardPositions)-1] - } + currentKeyBase = currentKeyBase[0 : len(currentKeyBase)-1] } - + keys = append(keys, currentKeyBase[0]) return keys } -func reverse(s [][]int) [][]int { - for i, j := 0, len(s)-1; i < j; i, j = i+1, j-1 { - s[i], s[j] = s[j], s[i] +// SortBySpecificity sorts an array of uints representing binary numbers, such that numbers with fewer 1-digits come first. +// Numbers with an equal amount of 1-digits are sorted by magnitude. +type sortBySpecificity []uint + +func (a sortBySpecificity) Len() int { return len(a) } +func (a sortBySpecificity) Swap(i, j int) { a[i], a[j] = a[j], a[i] } +func (a sortBySpecificity) Less(i, j int) bool { + onesI := bits.OnesCount(a[i]) + onesJ := bits.OnesCount(a[j]) + if onesI < onesJ { + return true } - return s -} - -func combinations(iterable []int, r int) (rt [][]int) { - pool := iterable - n := len(pool) - - if r > n { - return - } - - indices := make([]int, r) - for i := range indices { - indices[i] = i - } - - result := make([]int, r) - for i, el := range indices { - result[i] = pool[el] + if onesI > onesJ { + return false } - s2 := make([]int, r) - copy(s2, result) - rt = append(rt, s2) - - for { - i := r - 1 - for ; i >= 0 && indices[i] == i+n-r; i -= 1 { - } - - if i < 0 { - return - } - - indices[i] += 1 - for j := i + 1; j < r; j += 1 { - indices[j] = indices[j-1] + 1 - } + return a[i] < a[j] +} - for ; i < len(indices); i += 1 { - result[i] = pool[indices[i]] - } - s2 = make([]int, r) - copy(s2, result) - rt = append(rt, s2) +func makeRange(min, max int) []uint { + a := make([]uint, max-min) + for i := range a { + a[i] = uint(min + i) } + return a } diff --git a/pkg/sourcekey/sourcekey_test.go b/pkg/sourcekey/sourcekey_test.go index 61cec5d..8b6fa89 100644 --- a/pkg/sourcekey/sourcekey_test.go +++ b/pkg/sourcekey/sourcekey_test.go @@ -1,82 +1,85 @@ -package sourcekey_test +package sourcekey import ( "testing" "github.com/stretchr/testify/require" - - "github.com/appuio/appuio-cloud-reporting/pkg/sourcekey" ) func TestParseInvalidKey(t *testing.T) { - _, err := sourcekey.Parse("appuio_cloud_storage:c-appuio-cloudscale-lpg-2") + _, err := Parse("appuio_cloud_storage:c-appuio-cloudscale-lpg-2") require.Error(t, err) } -func TestParseWithClass(t *testing.T) { - k, err := sourcekey.Parse("appuio_cloud_storage:c-appuio-cloudscale-lpg-2:acme-corp:sparkling-sound-1234:ssd") +func TestParseWithclass(t *testing.T) { + k, err := Parse("appuio_cloud_storage:c-appuio-cloudscale-lpg-2:acme-corp:sparkling-sound-1234:ssd") require.NoError(t, err) - require.Equal(t, k, sourcekey.SourceKey{ - Query: "appuio_cloud_storage", - Zone: "c-appuio-cloudscale-lpg-2", - Tenant: "acme-corp", - Namespace: "sparkling-sound-1234", - Class: "ssd", - }) + require.Equal(t, SourceKey{ + parts: []string{"appuio_cloud_storage", "c-appuio-cloudscale-lpg-2", "acme-corp", "sparkling-sound-1234", "ssd"}, + }, k) } -func TestParseWithoutClass(t *testing.T) { - k, err := sourcekey.Parse("appuio_cloud_storage:c-appuio-cloudscale-lpg-2:acme-corp:sparkling-sound-1234") +func TestParseWithoutclass(t *testing.T) { + k, err := Parse("appuio_cloud_storage:c-appuio-cloudscale-lpg-2:acme-corp:sparkling-sound-1234") require.NoError(t, err) - require.Equal(t, k, sourcekey.SourceKey{ - Query: "appuio_cloud_storage", - Zone: "c-appuio-cloudscale-lpg-2", - Tenant: "acme-corp", - Namespace: "sparkling-sound-1234", - }) + require.Equal(t, SourceKey{ + parts: []string{"appuio_cloud_storage", "c-appuio-cloudscale-lpg-2", "acme-corp", "sparkling-sound-1234"}, + }, k) } -func TestParseWithEmptyClass(t *testing.T) { - k, err := sourcekey.Parse("appuio_cloud_storage:c-appuio-cloudscale-lpg-2:acme-corp:sparkling-sound-1234:") +func TestParseWithEmptyclass(t *testing.T) { + k, err := Parse("appuio_cloud_storage:c-appuio-cloudscale-lpg-2:acme-corp:sparkling-sound-1234:") require.NoError(t, err) - require.Equal(t, k, sourcekey.SourceKey{ - Query: "appuio_cloud_storage", - Zone: "c-appuio-cloudscale-lpg-2", - Tenant: "acme-corp", - Namespace: "sparkling-sound-1234", - }) + require.Equal(t, SourceKey{ + parts: []string{"appuio_cloud_storage", "c-appuio-cloudscale-lpg-2", "acme-corp", "sparkling-sound-1234"}, + }, k) } -func TestStringWithClass(t *testing.T) { - key := sourcekey.SourceKey{ - Query: "appuio_cloud_storage", - Zone: "c-appuio-cloudscale-lpg-2", - Tenant: "acme-corp", - Namespace: "sparkling-sound-1234", - Class: "ssd", +func TestStringWithclass(t *testing.T) { + key := SourceKey{ + parts: []string{"appuio_cloud_storage", "c-appuio-cloudscale-lpg-2", "acme-corp", "sparkling-sound-1234", "ssd"}, } require.Equal(t, "appuio_cloud_storage:c-appuio-cloudscale-lpg-2:acme-corp:sparkling-sound-1234:ssd", key.String()) } -func TestStringWithoutClass(t *testing.T) { - key := sourcekey.SourceKey{ - Query: "appuio_cloud_storage", - Zone: "c-appuio-cloudscale-lpg-2", - Tenant: "acme-corp", - Namespace: "sparkling-sound-1234", +func TestStringWithoutclass(t *testing.T) { + key := SourceKey{ + parts: []string{"appuio_cloud_storage", "c-appuio-cloudscale-lpg-2", "acme-corp", "sparkling-sound-1234"}, } require.Equal(t, "appuio_cloud_storage:c-appuio-cloudscale-lpg-2:acme-corp:sparkling-sound-1234", key.String()) } -func TestGenerateSourceKeysWithoutClass(t *testing.T) { - keys := sourcekey.SourceKey{ - Query: "appuio_cloud_storage", - Zone: "c-appuio-cloudscale-lpg-2", - Tenant: "acme-corp", - Namespace: "sparkling-sound-1234", +func TestGenerateSourceKeysWithoutclass(t *testing.T) { + keys := SourceKey{ + parts: []string{"appuio_cloud_storage", "c-appuio-cloudscale-lpg-2", "acme-corp", "sparkling-sound-1234"}, }.LookupKeys() - require.Equal(t, keys, []string{ + require.Equal(t, []string{ + "appuio_cloud_storage:c-appuio-cloudscale-lpg-2:acme-corp:sparkling-sound-1234", + "appuio_cloud_storage:c-appuio-cloudscale-lpg-2:*:sparkling-sound-1234", + "appuio_cloud_storage:*:acme-corp:sparkling-sound-1234", + "appuio_cloud_storage:*:*:sparkling-sound-1234", + "appuio_cloud_storage:c-appuio-cloudscale-lpg-2:acme-corp", + "appuio_cloud_storage:*:acme-corp", + "appuio_cloud_storage:c-appuio-cloudscale-lpg-2", + "appuio_cloud_storage", + }, keys) +} + +func TestGenerateSourceKeysWithclass(t *testing.T) { + keys := SourceKey{ + parts: []string{"appuio_cloud_storage", "c-appuio-cloudscale-lpg-2", "acme-corp", "sparkling-sound-1234", "ssd"}, + }.LookupKeys() + + require.Equal(t, []string{ + "appuio_cloud_storage:c-appuio-cloudscale-lpg-2:acme-corp:sparkling-sound-1234:ssd", + "appuio_cloud_storage:c-appuio-cloudscale-lpg-2:acme-corp:*:ssd", + "appuio_cloud_storage:c-appuio-cloudscale-lpg-2:*:sparkling-sound-1234:ssd", + "appuio_cloud_storage:*:acme-corp:sparkling-sound-1234:ssd", + "appuio_cloud_storage:c-appuio-cloudscale-lpg-2:*:*:ssd", + "appuio_cloud_storage:*:acme-corp:*:ssd", + "appuio_cloud_storage:*:*:sparkling-sound-1234:ssd", + "appuio_cloud_storage:*:*:*:ssd", "appuio_cloud_storage:c-appuio-cloudscale-lpg-2:acme-corp:sparkling-sound-1234", "appuio_cloud_storage:c-appuio-cloudscale-lpg-2:*:sparkling-sound-1234", "appuio_cloud_storage:*:acme-corp:sparkling-sound-1234", @@ -85,19 +88,31 @@ func TestGenerateSourceKeysWithoutClass(t *testing.T) { "appuio_cloud_storage:*:acme-corp", "appuio_cloud_storage:c-appuio-cloudscale-lpg-2", "appuio_cloud_storage", - }) + }, keys) } -func TestGenerateSourceKeysWithClass(t *testing.T) { - keys := sourcekey.SourceKey{ - Query: "appuio_cloud_storage", - Zone: "c-appuio-cloudscale-lpg-2", - Tenant: "acme-corp", - Namespace: "sparkling-sound-1234", - Class: "ssd", +func TestGenerateSourceKeysWithSixElements(t *testing.T) { + keys := SourceKey{ + parts: []string{"appuio_cloud_storage", "c-appuio-cloudscale-lpg-2", "acme-corp", "sparkling-sound-1234", "ssd", "exoscale"}, }.LookupKeys() - require.Equal(t, keys, []string{ + require.Equal(t, []string{ + "appuio_cloud_storage:c-appuio-cloudscale-lpg-2:acme-corp:sparkling-sound-1234:ssd:exoscale", + "appuio_cloud_storage:c-appuio-cloudscale-lpg-2:acme-corp:sparkling-sound-1234:*:exoscale", + "appuio_cloud_storage:c-appuio-cloudscale-lpg-2:acme-corp:*:ssd:exoscale", + "appuio_cloud_storage:c-appuio-cloudscale-lpg-2:*:sparkling-sound-1234:ssd:exoscale", + "appuio_cloud_storage:*:acme-corp:sparkling-sound-1234:ssd:exoscale", + "appuio_cloud_storage:c-appuio-cloudscale-lpg-2:acme-corp:*:*:exoscale", + "appuio_cloud_storage:c-appuio-cloudscale-lpg-2:*:sparkling-sound-1234:*:exoscale", + "appuio_cloud_storage:c-appuio-cloudscale-lpg-2:*:*:ssd:exoscale", + "appuio_cloud_storage:*:acme-corp:sparkling-sound-1234:*:exoscale", + "appuio_cloud_storage:*:acme-corp:*:ssd:exoscale", + "appuio_cloud_storage:*:*:sparkling-sound-1234:ssd:exoscale", + "appuio_cloud_storage:c-appuio-cloudscale-lpg-2:*:*:*:exoscale", + "appuio_cloud_storage:*:acme-corp:*:*:exoscale", + "appuio_cloud_storage:*:*:sparkling-sound-1234:*:exoscale", + "appuio_cloud_storage:*:*:*:ssd:exoscale", + "appuio_cloud_storage:*:*:*:*:exoscale", "appuio_cloud_storage:c-appuio-cloudscale-lpg-2:acme-corp:sparkling-sound-1234:ssd", "appuio_cloud_storage:c-appuio-cloudscale-lpg-2:acme-corp:*:ssd", "appuio_cloud_storage:c-appuio-cloudscale-lpg-2:*:sparkling-sound-1234:ssd", @@ -114,5 +129,5 @@ func TestGenerateSourceKeysWithClass(t *testing.T) { "appuio_cloud_storage:*:acme-corp", "appuio_cloud_storage:c-appuio-cloudscale-lpg-2", "appuio_cloud_storage", - }) + }, keys) }