diff options
author | Francis Lavoie <[email protected]> | 2024-03-05 18:24:32 -0500 |
---|---|---|
committer | GitHub <[email protected]> | 2024-03-05 16:24:32 -0700 |
commit | 01d5568b20408a7f72fb53095e2e146f0c39672a (patch) | |
tree | 858bd2129c48e24cce117a08cc3f711cbb6c15c1 | |
parent | 1f4a6fa7e7d60bbcd9e29ca35072a76d61cf84c3 (diff) | |
download | caddy-01d5568b20408a7f72fb53095e2e146f0c39672a.tar.gz caddy-01d5568b20408a7f72fb53095e2e146f0c39672a.zip |
logging: Implement `append` encoder, allow flatter filters config (#6069)
* logging: Implement `add` encoder
* Allow flatter config structure for `filter` & `add`
* Rename to append
* govulncheck was unhappy
-rw-r--r-- | .github/workflows/ci.yml | 2 | ||||
-rw-r--r-- | .github/workflows/cross-build.yml | 2 | ||||
-rw-r--r-- | .github/workflows/lint.yml | 4 | ||||
-rw-r--r-- | caddytest/integration/caddyfile_adapt/log_append_encoder.caddyfiletest | 63 | ||||
-rw-r--r-- | caddytest/integration/caddyfile_adapt/log_filters.caddyfiletest | 32 | ||||
-rw-r--r-- | modules/logging/appendencoder.go | 357 | ||||
-rw-r--r-- | modules/logging/filterencoder.go | 50 |
7 files changed, 476 insertions, 34 deletions
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3fe65a653..309ef7935 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -33,7 +33,7 @@ jobs: GO_SEMVER: '~1.21.0' - go: '1.22' - GO_SEMVER: '~1.22.0' + GO_SEMVER: '~1.22.1' # Set some variables per OS, usable via ${{ matrix.VAR }} # OS_LABEL: the VM label from GitHub Actions (see https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners/about-github-hosted-runners#standard-github-hosted-runners-for-public-repositories) diff --git a/.github/workflows/cross-build.yml b/.github/workflows/cross-build.yml index b097f75ed..676607d0e 100644 --- a/.github/workflows/cross-build.yml +++ b/.github/workflows/cross-build.yml @@ -35,7 +35,7 @@ jobs: # Set the minimum Go patch version for the given Go minor # Usable via ${{ matrix.GO_SEMVER }} - go: '1.22' - GO_SEMVER: '~1.22.0' + GO_SEMVER: '~1.22.1' runs-on: ubuntu-latest continue-on-error: true diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 918734751..bfb91dc66 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -43,7 +43,7 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-go@v5 with: - go-version: '~1.22.0' + go-version: '~1.22.1' check-latest: true - name: golangci-lint @@ -66,5 +66,5 @@ jobs: - name: govulncheck uses: golang/govulncheck-action@v1 with: - go-version-input: '~1.22.0' + go-version-input: '~1.22.1' check-latest: true diff --git a/caddytest/integration/caddyfile_adapt/log_append_encoder.caddyfiletest b/caddytest/integration/caddyfile_adapt/log_append_encoder.caddyfiletest new file mode 100644 index 000000000..88a6cd6be --- /dev/null +++ b/caddytest/integration/caddyfile_adapt/log_append_encoder.caddyfiletest @@ -0,0 +1,63 @@ +{ + log { + format append { + wrap json + fields { + wrap "foo" + } + env {env.EXAMPLE} + int 1 + float 1.1 + bool true + string "string" + } + } +} + +:80 { + respond "Hello, World!" +} +---------- +{ + "logging": { + "logs": { + "default": { + "encoder": { + "fields": { + "bool": true, + "env": "{env.EXAMPLE}", + "float": 1.1, + "int": 1, + "string": "string", + "wrap": "foo" + }, + "format": "append", + "wrap": { + "format": "json" + } + } + } + } + }, + "apps": { + "http": { + "servers": { + "srv0": { + "listen": [ + ":80" + ], + "routes": [ + { + "handle": [ + { + "body": "Hello, World!", + "handler": "static_response" + } + ] + } + ] + } + } + } + } +}
\ No newline at end of file diff --git a/caddytest/integration/caddyfile_adapt/log_filters.caddyfiletest b/caddytest/integration/caddyfile_adapt/log_filters.caddyfiletest index 28524a346..1b2fc2e50 100644 --- a/caddytest/integration/caddyfile_adapt/log_filters.caddyfiletest +++ b/caddytest/integration/caddyfile_adapt/log_filters.caddyfiletest @@ -4,27 +4,31 @@ log { output stdout format filter { wrap console + + # long form, with "fields" wrapper fields { uri query { replace foo REDACTED delete bar hash baz } - request>headers>Authorization replace REDACTED - request>headers>Server delete - request>headers>Cookie cookie { - replace foo REDACTED - delete bar - hash baz - } - request>remote_ip ip_mask { - ipv4 24 - ipv6 32 - } - request>client_ip ip_mask 16 32 - request>headers>Regexp regexp secret REDACTED - request>headers>Hash hash } + + # short form, flatter structure + request>headers>Authorization replace REDACTED + request>headers>Server delete + request>headers>Cookie cookie { + replace foo REDACTED + delete bar + hash baz + } + request>remote_ip ip_mask { + ipv4 24 + ipv6 32 + } + request>client_ip ip_mask 16 32 + request>headers>Regexp regexp secret REDACTED + request>headers>Hash hash } } ---------- diff --git a/modules/logging/appendencoder.go b/modules/logging/appendencoder.go new file mode 100644 index 000000000..63bd532d0 --- /dev/null +++ b/modules/logging/appendencoder.go @@ -0,0 +1,357 @@ +// Copyright 2015 Matthew Holt and The Caddy Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package logging + +import ( + "encoding/json" + "fmt" + "os" + "strings" + "time" + + "go.uber.org/zap" + "go.uber.org/zap/buffer" + "go.uber.org/zap/zapcore" + "golang.org/x/term" + + "github.com/caddyserver/caddy/v2" + "github.com/caddyserver/caddy/v2/caddyconfig" + "github.com/caddyserver/caddy/v2/caddyconfig/caddyfile" +) + +func init() { + caddy.RegisterModule(AppendEncoder{}) +} + +// AppendEncoder can be used to add fields to all log entries +// that pass through it. It is a wrapper around another +// encoder, which it uses to actually encode the log entries. +// It is most useful for adding information about the Caddy +// instance that is producing the log entries, possibly via +// an environment variable. +type AppendEncoder struct { + // The underlying encoder that actually encodes the + // log entries. If not specified, defaults to "json", + // unless the output is a terminal, in which case + // it defaults to "console". + WrappedRaw json.RawMessage `json:"wrap,omitempty" caddy:"namespace=caddy.logging.encoders inline_key=format"` + + // A map of field names to their values. The values + // can be global placeholders (e.g. env vars), or constants. + // Note that the encoder does not run as part of an HTTP + // request context, so request placeholders are not available. + Fields map[string]any `json:"fields,omitempty"` + + wrapped zapcore.Encoder + repl *caddy.Replacer + + wrappedIsDefault bool + ctx caddy.Context +} + +// CaddyModule returns the Caddy module information. +func (AppendEncoder) CaddyModule() caddy.ModuleInfo { + return caddy.ModuleInfo{ + ID: "caddy.logging.encoders.append", + New: func() caddy.Module { return new(AppendEncoder) }, + } +} + +// Provision sets up the encoder. +func (fe *AppendEncoder) Provision(ctx caddy.Context) error { + fe.ctx = ctx + fe.repl = caddy.NewReplacer() + + if fe.WrappedRaw == nil { + // if wrap is not specified, default to JSON + fe.wrapped = &JSONEncoder{} + if p, ok := fe.wrapped.(caddy.Provisioner); ok { + if err := p.Provision(ctx); err != nil { + return fmt.Errorf("provisioning fallback encoder module: %v", err) + } + } + fe.wrappedIsDefault = true + } else { + // set up wrapped encoder + val, err := ctx.LoadModule(fe, "WrappedRaw") + if err != nil { + return fmt.Errorf("loading fallback encoder module: %v", err) + } + fe.wrapped = val.(zapcore.Encoder) + } + + return nil +} + +// ConfigureDefaultFormat will set the default format to "console" +// if the writer is a terminal. If already configured, it passes +// through the writer so a deeply nested encoder can configure +// its own default format. +func (fe *AppendEncoder) ConfigureDefaultFormat(wo caddy.WriterOpener) error { + if !fe.wrappedIsDefault { + if cfd, ok := fe.wrapped.(caddy.ConfiguresFormatterDefault); ok { + return cfd.ConfigureDefaultFormat(wo) + } + return nil + } + + if caddy.IsWriterStandardStream(wo) && term.IsTerminal(int(os.Stderr.Fd())) { + fe.wrapped = &ConsoleEncoder{} + if p, ok := fe.wrapped.(caddy.Provisioner); ok { + if err := p.Provision(fe.ctx); err != nil { + return fmt.Errorf("provisioning fallback encoder module: %v", err) + } + } + } + return nil +} + +// UnmarshalCaddyfile sets up the module from Caddyfile tokens. Syntax: +// +// append { +// wrap <another encoder> +// fields { +// <field> <value> +// } +// <field> <value> +// } +func (fe *AppendEncoder) UnmarshalCaddyfile(d *caddyfile.Dispenser) error { + d.Next() // consume encoder name + + // parse a field + parseField := func() error { + if fe.Fields == nil { + fe.Fields = make(map[string]any) + } + field := d.Val() + if !d.NextArg() { + return d.ArgErr() + } + fe.Fields[field] = d.ScalarVal() + if d.NextArg() { + return d.ArgErr() + } + return nil + } + + for d.NextBlock(0) { + switch d.Val() { + case "wrap": + if !d.NextArg() { + return d.ArgErr() + } + moduleName := d.Val() + moduleID := "caddy.logging.encoders." + moduleName + unm, err := caddyfile.UnmarshalModule(d, moduleID) + if err != nil { + return err + } + enc, ok := unm.(zapcore.Encoder) + if !ok { + return d.Errf("module %s (%T) is not a zapcore.Encoder", moduleID, unm) + } + fe.WrappedRaw = caddyconfig.JSONModuleObject(enc, "format", moduleName, nil) + + case "fields": + for nesting := d.Nesting(); d.NextBlock(nesting); { + err := parseField() + if err != nil { + return err + } + } + + default: + // if unknown, assume it's a field so that + // the config can be flat + err := parseField() + if err != nil { + return err + } + } + } + return nil +} + +// AddArray is part of the zapcore.ObjectEncoder interface. +func (fe AppendEncoder) AddArray(key string, marshaler zapcore.ArrayMarshaler) error { + return fe.wrapped.AddArray(key, marshaler) +} + +// AddObject is part of the zapcore.ObjectEncoder interface. +func (fe AppendEncoder) AddObject(key string, marshaler zapcore.ObjectMarshaler) error { + return fe.wrapped.AddObject(key, marshaler) +} + +// AddBinary is part of the zapcore.ObjectEncoder interface. +func (fe AppendEncoder) AddBinary(key string, value []byte) { + fe.wrapped.AddBinary(key, value) +} + +// AddByteString is part of the zapcore.ObjectEncoder interface. +func (fe AppendEncoder) AddByteString(key string, value []byte) { + fe.wrapped.AddByteString(key, value) +} + +// AddBool is part of the zapcore.ObjectEncoder interface. +func (fe AppendEncoder) AddBool(key string, value bool) { + fe.wrapped.AddBool(key, value) +} + +// AddComplex128 is part of the zapcore.ObjectEncoder interface. +func (fe AppendEncoder) AddComplex128(key string, value complex128) { + fe.wrapped.AddComplex128(key, value) +} + +// AddComplex64 is part of the zapcore.ObjectEncoder interface. +func (fe AppendEncoder) AddComplex64(key string, value complex64) { + fe.wrapped.AddComplex64(key, value) +} + +// AddDuration is part of the zapcore.ObjectEncoder interface. +func (fe AppendEncoder) AddDuration(key string, value time.Duration) { + fe.wrapped.AddDuration(key, value) +} + +// AddFloat64 is part of the zapcore.ObjectEncoder interface. +func (fe AppendEncoder) AddFloat64(key string, value float64) { + fe.wrapped.AddFloat64(key, value) +} + +// AddFloat32 is part of the zapcore.ObjectEncoder interface. +func (fe AppendEncoder) AddFloat32(key string, value float32) { + fe.wrapped.AddFloat32(key, value) +} + +// AddInt is part of the zapcore.ObjectEncoder interface. +func (fe AppendEncoder) AddInt(key string, value int) { + fe.wrapped.AddInt(key, value) +} + +// AddInt64 is part of the zapcore.ObjectEncoder interface. +func (fe AppendEncoder) AddInt64(key string, value int64) { + fe.wrapped.AddInt64(key, value) +} + +// AddInt32 is part of the zapcore.ObjectEncoder interface. +func (fe AppendEncoder) AddInt32(key string, value int32) { + fe.wrapped.AddInt32(key, value) +} + +// AddInt16 is part of the zapcore.ObjectEncoder interface. +func (fe AppendEncoder) AddInt16(key string, value int16) { + fe.wrapped.AddInt16(key, value) +} + +// AddInt8 is part of the zapcore.ObjectEncoder interface. +func (fe AppendEncoder) AddInt8(key string, value int8) { + fe.wrapped.AddInt8(key, value) +} + +// AddString is part of the zapcore.ObjectEncoder interface. +func (fe AppendEncoder) AddString(key, value string) { + fe.wrapped.AddString(key, value) +} + +// AddTime is part of the zapcore.ObjectEncoder interface. +func (fe AppendEncoder) AddTime(key string, value time.Time) { + fe.wrapped.AddTime(key, value) +} + +// AddUint is part of the zapcore.ObjectEncoder interface. +func (fe AppendEncoder) AddUint(key string, value uint) { + fe.wrapped.AddUint(key, value) +} + +// AddUint64 is part of the zapcore.ObjectEncoder interface. +func (fe AppendEncoder) AddUint64(key string, value uint64) { + fe.wrapped.AddUint64(key, value) +} + +// AddUint32 is part of the zapcore.ObjectEncoder interface. +func (fe AppendEncoder) AddUint32(key string, value uint32) { + fe.wrapped.AddUint32(key, value) +} + +// AddUint16 is part of the zapcore.ObjectEncoder interface. +func (fe AppendEncoder) AddUint16(key string, value uint16) { + fe.wrapped.AddUint16(key, value) +} + +// AddUint8 is part of the zapcore.ObjectEncoder interface. +func (fe AppendEncoder) AddUint8(key string, value uint8) { + fe.wrapped.AddUint8(key, value) +} + +// AddUintptr is part of the zapcore.ObjectEncoder interface. +func (fe AppendEncoder) AddUintptr(key string, value uintptr) { + fe.wrapped.AddUintptr(key, value) +} + +// AddReflected is part of the zapcore.ObjectEncoder interface. +func (fe AppendEncoder) AddReflected(key string, value any) error { + return fe.wrapped.AddReflected(key, value) +} + +// OpenNamespace is part of the zapcore.ObjectEncoder interface. +func (fe AppendEncoder) OpenNamespace(key string) { + fe.wrapped.OpenNamespace(key) +} + +// Clone is part of the zapcore.ObjectEncoder interface. +func (fe AppendEncoder) Clone() zapcore.Encoder { + return AppendEncoder{ + Fields: fe.Fields, + wrapped: fe.wrapped.Clone(), + repl: fe.repl, + } +} + +// EncodeEntry partially implements the zapcore.Encoder interface. +func (fe AppendEncoder) EncodeEntry(ent zapcore.Entry, fields []zapcore.Field) (*buffer.Buffer, error) { + fe.wrapped = fe.wrapped.Clone() + for _, field := range fields { + field.AddTo(fe) + } + + // append fields from config + for key, value := range fe.Fields { + // if the value is a string + if str, ok := value.(string); ok { + isPlaceholder := strings.HasPrefix(str, "{") && + strings.HasSuffix(str, "}") && + strings.Count(str, "{") == 1 + if isPlaceholder { + // and it looks like a placeholder, evaluate it + replaced, _ := fe.repl.Get(strings.Trim(str, "{}")) + zap.Any(key, replaced).AddTo(fe) + } else { + // just use the string as-is + zap.String(key, str).AddTo(fe) + } + } else { + // not a string, so use the value as any + zap.Any(key, value).AddTo(fe) + } + } + + return fe.wrapped.EncodeEntry(ent, nil) +} + +// Interface guards +var ( + _ zapcore.Encoder = (*AppendEncoder)(nil) + _ caddyfile.Unmarshaler = (*AppendEncoder)(nil) + _ caddy.ConfiguresFormatterDefault = (*AppendEncoder)(nil) +) diff --git a/modules/logging/filterencoder.go b/modules/logging/filterencoder.go index 9b1895d79..c46df0788 100644 --- a/modules/logging/filterencoder.go +++ b/modules/logging/filterencoder.go @@ -145,9 +145,36 @@ func (fe *FilterEncoder) ConfigureDefaultFormat(wo caddy.WriterOpener) error { // <filter options> // } // } +// <field> <filter> { +// <filter options> +// } // } func (fe *FilterEncoder) UnmarshalCaddyfile(d *caddyfile.Dispenser) error { d.Next() // consume encoder name + + // parse a field + parseField := func() error { + if fe.FieldsRaw == nil { + fe.FieldsRaw = make(map[string]json.RawMessage) + } + field := d.Val() + if !d.NextArg() { + return d.ArgErr() + } + filterName := d.Val() + moduleID := "caddy.logging.encoders.filter." + filterName + unm, err := caddyfile.UnmarshalModule(d, moduleID) + if err != nil { + return err + } + filter, ok := unm.(LogFieldFilter) + if !ok { + return d.Errf("module %s (%T) is not a logging.LogFieldFilter", moduleID, unm) + } + fe.FieldsRaw[field] = caddyconfig.JSONModuleObject(filter, "filter", filterName, nil) + return nil + } + for d.NextBlock(0) { switch d.Val() { case "wrap": @@ -168,28 +195,19 @@ func (fe *FilterEncoder) UnmarshalCaddyfile(d *caddyfile.Dispenser) error { case "fields": for nesting := d.Nesting(); d.NextBlock(nesting); { - field := d.Val() - if !d.NextArg() { - return d.ArgErr() - } - filterName := d.Val() - moduleID := "caddy.logging.encoders.filter." + filterName - unm, err := caddyfile.UnmarshalModule(d, moduleID) + err := parseField() if err != nil { return err } - filter, ok := unm.(LogFieldFilter) - if !ok { - return d.Errf("module %s (%T) is not a logging.LogFieldFilter", moduleID, unm) - } - if fe.FieldsRaw == nil { - fe.FieldsRaw = make(map[string]json.RawMessage) - } - fe.FieldsRaw[field] = caddyconfig.JSONModuleObject(filter, "filter", filterName, nil) } default: - return d.Errf("unrecognized subdirective %s", d.Val()) + // if unknown, assume it's a field so that + // the config can be flat + err := parseField() + if err != nil { + return err + } } } return nil |