From d392893cd73dc00c927f342778f6dca9628d328e Mon Sep 17 00:00:00 2001 From: Bjørn Erik Pedersen Date: Wed, 9 Jun 2021 10:58:18 +0200 Subject: Misc config loading fixes The main motivation behind this is simplicity and correctnes, but the new small config library is also faster: ``` BenchmarkDefaultConfigProvider/Viper-16 252418 4546 ns/op 2720 B/op 30 allocs/op BenchmarkDefaultConfigProvider/Custom-16 450756 2651 ns/op 1008 B/op 6 allocs/op ``` Fixes #8633 Fixes #8618 Fixes #8630 Updates #8591 Closes #6680 Closes #5192 --- common/maps/maps.go | 77 +++++++++++++---------- common/maps/maps_test.go | 2 +- common/maps/params.go | 148 +++++++++++++++++++++++++++++++++++++++++++++ common/maps/params_test.go | 87 ++++++++++++++++++++++++++ 4 files changed, 280 insertions(+), 34 deletions(-) (limited to 'common/maps') diff --git a/common/maps/maps.go b/common/maps/maps.go index 41d9b6e15..5fb079009 100644 --- a/common/maps/maps.go +++ b/common/maps/maps.go @@ -18,53 +18,65 @@ import ( "strings" "github.com/gobwas/glob" - "github.com/spf13/cast" ) -// ToLower makes all the keys in the given map lower cased and will do so -// recursively. -// Notes: -// * This will modify the map given. -// * Any nested map[interface{}]interface{} will be converted to Params. -func ToLower(m Params) { - for k, v := range m { - var retyped bool - switch v.(type) { - case map[interface{}]interface{}: - var p Params = cast.ToStringMap(v) - v = p - ToLower(p) - retyped = true - case map[string]interface{}: - var p Params = v.(map[string]interface{}) - v = p - ToLower(p) - retyped = true +// ToStringMapE converts in to map[string]interface{}. +func ToStringMapE(in interface{}) (map[string]interface{}, error) { + switch vv := in.(type) { + case Params: + return vv, nil + case map[string]string: + var m = map[string]interface{}{} + for k, v := range vv { + m[k] = v } + return m, nil - lKey := strings.ToLower(k) - if retyped || k != lKey { - delete(m, k) - m[lKey] = v - } + default: + return cast.ToStringMapE(in) } } -func ToStringMapE(in interface{}) (map[string]interface{}, error) { - switch in.(type) { - case Params: - return in.(Params), nil - default: - return cast.ToStringMapE(in) +// ToParamsAndPrepare converts in to Params and prepares it for use. +// See PrepareParams. +func ToParamsAndPrepare(in interface{}) (Params, bool) { + m, err := ToStringMapE(in) + if err != nil { + return nil, false } + PrepareParams(m) + return m, true } +// ToStringMap converts in to map[string]interface{}. func ToStringMap(in interface{}) map[string]interface{} { m, _ := ToStringMapE(in) return m } +// ToStringMapStringE converts in to map[string]string. +func ToStringMapStringE(in interface{}) (map[string]string, error) { + m, err := ToStringMapE(in) + if err != nil { + return nil, err + } + return cast.ToStringMapStringE(m) +} + +// ToStringMapString converts in to map[string]string. +func ToStringMapString(in interface{}) map[string]string { + m, _ := ToStringMapStringE(in) + return m +} + +// ToStringMapBool converts in to bool. +func ToStringMapBool(in interface{}) map[string]bool { + m, _ := ToStringMapE(in) + return cast.ToStringMapBool(m) +} + +// ToSliceStringMap converts in to []map[string]interface{}. func ToSliceStringMap(in interface{}) ([]map[string]interface{}, error) { switch v := in.(type) { case []map[string]interface{}: @@ -127,9 +139,8 @@ func (KeyRenamer) keyPath(k1, k2 string) string { k1, k2 = strings.ToLower(k1), strings.ToLower(k2) if k1 == "" { return k2 - } else { - return k1 + "/" + k2 } + return k1 + "/" + k2 } func (r KeyRenamer) renamePath(parentKeyPath string, m map[string]interface{}) { diff --git a/common/maps/maps_test.go b/common/maps/maps_test.go index 7e527aac5..dbe97a15a 100644 --- a/common/maps/maps_test.go +++ b/common/maps/maps_test.go @@ -67,7 +67,7 @@ func TestToLower(t *testing.T) { for i, test := range tests { t.Run(fmt.Sprint(i), func(t *testing.T) { // ToLower modifies input. - ToLower(test.input) + PrepareParams(test.input) if !reflect.DeepEqual(test.expected, test.input) { t.Errorf("[%d] Expected\n%#v, got\n%#v\n", i, test.expected, test.input) } diff --git a/common/maps/params.go b/common/maps/params.go index 4c881093c..7e94d593b 100644 --- a/common/maps/params.go +++ b/common/maps/params.go @@ -14,6 +14,7 @@ package maps import ( + "fmt" "strings" "github.com/spf13/cast" @@ -29,6 +30,95 @@ func (p Params) Get(indices ...string) interface{} { return v } +// Set overwrites values in p with values in pp for common or new keys. +// This is done recursively. +func (p Params) Set(pp Params) { + for k, v := range pp { + vv, found := p[k] + if !found { + p[k] = v + } else { + switch vvv := vv.(type) { + case Params: + if pv, ok := v.(Params); ok { + vvv.Set(pv) + } else { + p[k] = v + } + default: + p[k] = v + } + } + } +} + +// Merge transfers values from pp to p for new keys. +// This is done recursively. +func (p Params) Merge(pp Params) { + p.merge("", pp) +} + +func (p Params) merge(ps ParamsMergeStrategy, pp Params) { + ns, found := p.GetMergeStrategy() + + var ms = ns + if !found && ps != "" { + ms = ps + } + + noUpdate := ms == ParamsMergeStrategyNone + noUpdate = noUpdate || (ps != "" && ps == ParamsMergeStrategyShallow) + + for k, v := range pp { + + if k == mergeStrategyKey { + continue + } + vv, found := p[k] + + if found { + // Key matches, if both sides are Params, we try to merge. + if vvv, ok := vv.(Params); ok { + if pv, ok := v.(Params); ok { + vvv.merge(ms, pv) + } + + } + + } else if !noUpdate { + p[k] = v + + } + + } +} + +func (p Params) GetMergeStrategy() (ParamsMergeStrategy, bool) { + if v, found := p[mergeStrategyKey]; found { + if s, ok := v.(ParamsMergeStrategy); ok { + return s, true + } + } + return ParamsMergeStrategyShallow, false +} + +func (p Params) DeleteMergeStrategy() bool { + if _, found := p[mergeStrategyKey]; found { + delete(p, mergeStrategyKey) + return true + } + return false +} + +func (p Params) SetDefaultMergeStrategy(s ParamsMergeStrategy) { + switch s { + case ParamsMergeStrategyDeep, ParamsMergeStrategyNone, ParamsMergeStrategyShallow: + default: + panic(fmt.Sprintf("invalid merge strategy %q", s)) + } + p[mergeStrategyKey] = s +} + func getNested(m map[string]interface{}, indices []string) (interface{}, string, map[string]interface{}) { if len(indices) == 0 { return nil, "", nil @@ -108,3 +198,61 @@ func GetNestedParamFn(keyStr, separator string, lookupFn func(key string) interf return nil, "", nil, nil } + +// ParamsMergeStrategy tells what strategy to use in Params.Merge. +type ParamsMergeStrategy string + +const ( + // Do not merge. + ParamsMergeStrategyNone ParamsMergeStrategy = "none" + // Only add new keys. + ParamsMergeStrategyShallow ParamsMergeStrategy = "shallow" + // Add new keys, merge existing. + ParamsMergeStrategyDeep ParamsMergeStrategy = "deep" + + mergeStrategyKey = "_merge" +) + +func toMergeStrategy(v interface{}) ParamsMergeStrategy { + s := ParamsMergeStrategy(cast.ToString(v)) + switch s { + case ParamsMergeStrategyDeep, ParamsMergeStrategyNone, ParamsMergeStrategyShallow: + return s + default: + return ParamsMergeStrategyDeep + } +} + +// PrepareParams +// * makes all the keys in the given map lower cased and will do so +// * This will modify the map given. +// * Any nested map[interface{}]interface{} will be converted to Params. +// * Any _merge value will be converted to proper type and value. +func PrepareParams(m Params) { + for k, v := range m { + var retyped bool + lKey := strings.ToLower(k) + if lKey == mergeStrategyKey { + v = toMergeStrategy(v) + retyped = true + } else { + switch v.(type) { + case map[interface{}]interface{}: + var p Params = cast.ToStringMap(v) + v = p + PrepareParams(p) + retyped = true + case map[string]interface{}: + var p Params = v.(map[string]interface{}) + v = p + PrepareParams(p) + retyped = true + } + } + + if retyped || k != lKey { + delete(m, k) + m[lKey] = v + } + } +} diff --git a/common/maps/params_test.go b/common/maps/params_test.go index df8cbf8d6..8859bb86b 100644 --- a/common/maps/params_test.go +++ b/common/maps/params_test.go @@ -69,3 +69,90 @@ func TestGetNestedParamFnNestedNewKey(t *testing.T) { c.Assert(nestedKey, qt.Equals, "new") c.Assert(owner, qt.DeepEquals, nested) } + +func TestParamsSetAndMerge(t *testing.T) { + c := qt.New(t) + + createParamsPair := func() (Params, Params) { + p1 := Params{"a": "av", "c": "cv", "nested": Params{"al2": "al2v", "cl2": "cl2v"}} + p2 := Params{"b": "bv", "a": "abv", "nested": Params{"bl2": "bl2v", "al2": "al2bv"}, mergeStrategyKey: ParamsMergeStrategyDeep} + return p1, p2 + } + + p1, p2 := createParamsPair() + + p1.Set(p2) + + c.Assert(p1, qt.DeepEquals, Params{ + "a": "abv", + "c": "cv", + "nested": Params{ + "al2": "al2bv", + "cl2": "cl2v", + "bl2": "bl2v", + }, + "b": "bv", + mergeStrategyKey: ParamsMergeStrategyDeep, + }) + + p1, p2 = createParamsPair() + + p1.Merge(p2) + + // Default is to do a shallow merge. + c.Assert(p1, qt.DeepEquals, Params{ + "c": "cv", + "nested": Params{ + "al2": "al2v", + "cl2": "cl2v", + }, + "b": "bv", + "a": "av", + }) + + p1, p2 = createParamsPair() + p1.SetDefaultMergeStrategy(ParamsMergeStrategyNone) + p1.Merge(p2) + p1.DeleteMergeStrategy() + + c.Assert(p1, qt.DeepEquals, Params{ + "a": "av", + "c": "cv", + "nested": Params{ + "al2": "al2v", + "cl2": "cl2v", + }, + }) + + p1, p2 = createParamsPair() + p1.SetDefaultMergeStrategy(ParamsMergeStrategyShallow) + p1.Merge(p2) + p1.DeleteMergeStrategy() + + c.Assert(p1, qt.DeepEquals, Params{ + "a": "av", + "c": "cv", + "nested": Params{ + "al2": "al2v", + "cl2": "cl2v", + }, + "b": "bv", + }) + + p1, p2 = createParamsPair() + p1.SetDefaultMergeStrategy(ParamsMergeStrategyDeep) + p1.Merge(p2) + p1.DeleteMergeStrategy() + + c.Assert(p1, qt.DeepEquals, Params{ + "nested": Params{ + "al2": "al2v", + "cl2": "cl2v", + "bl2": "bl2v", + }, + "b": "bv", + "a": "av", + "c": "cv", + }) + +} -- cgit v1.2.3