1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
|
// Copyright 2015 Matthew Holt and The Caddy Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Even though the filename ends in _unix.go, we still have to specify the
// build constraint here, because the filename convention only works for
// literal GOOS values, and "unix" is a shortcut unique to build tags.
//go:build unix
package caddy
import (
"context"
"errors"
"io/fs"
"net"
"sync/atomic"
"syscall"
"go.uber.org/zap"
"golang.org/x/sys/unix"
)
// reuseUnixSocket copies and reuses the unix domain socket (UDS) if we already
// have it open; if not, unlink it so we can have it. No-op if not a unix network.
func reuseUnixSocket(network, addr string) (any, error) {
if !IsUnixNetwork(network) {
return nil, nil
}
socketKey := listenerKey(network, addr)
socket, exists := unixSockets[socketKey]
if exists {
// make copy of file descriptor
socketFile, err := socket.File() // does dup() deep down
if err != nil {
return nil, err
}
// use copied fd to make new Listener or PacketConn, then replace
// it in the map so that future copies always come from the most
// recent fd (as the previous ones will be closed, and we'd get
// "use of closed network connection" errors) -- note that we
// preserve the *pointer* to the counter (not just the value) so
// that all socket wrappers will refer to the same value
switch unixSocket := socket.(type) {
case *unixListener:
ln, err := net.FileListener(socketFile)
if err != nil {
return nil, err
}
atomic.AddInt32(unixSocket.count, 1)
unixSockets[socketKey] = &unixListener{ln.(*net.UnixListener), socketKey, unixSocket.count}
case *unixConn:
pc, err := net.FilePacketConn(socketFile)
if err != nil {
return nil, err
}
atomic.AddInt32(unixSocket.count, 1)
unixSockets[socketKey] = &unixConn{pc.(*net.UnixConn), addr, socketKey, unixSocket.count}
}
return unixSockets[socketKey], nil
}
// from what I can tell after some quick research, it's quite common for programs to
// leave their socket file behind after they close, so the typical pattern is to
// unlink it before you bind to it -- this is often crucial if the last program using
// it was killed forcefully without a chance to clean up the socket, but there is a
// race, as the comment in net.UnixListener.close() explains... oh well, I guess?
if err := syscall.Unlink(addr); err != nil && !errors.Is(err, fs.ErrNotExist) {
return nil, err
}
return nil, nil
}
func listenTCPOrUnix(ctx context.Context, lnKey string, network, address string, config net.ListenConfig) (net.Listener, error) {
// wrap any Control function set by the user so we can also add our reusePort control without clobbering theirs
oldControl := config.Control
config.Control = func(network, address string, c syscall.RawConn) error {
if oldControl != nil {
if err := oldControl(network, address, c); err != nil {
return err
}
}
return reusePort(network, address, c)
}
// even though SO_REUSEPORT lets us bind the socket multiple times,
// we still put it in the listenerPool so we can count how many
// configs are using this socket; necessary to ensure we can know
// whether to enforce shutdown delays, for example (see #5393).
ln, err := config.Listen(ctx, network, address)
if err == nil {
listenerPool.LoadOrStore(lnKey, nil)
}
// if new listener is a unix socket, make sure we can reuse it later
// (we do our own "unlink on close" -- not required, but more tidy)
one := int32(1)
if unix, ok := ln.(*net.UnixListener); ok {
unix.SetUnlinkOnClose(false)
ln = &unixListener{unix, lnKey, &one}
unixSockets[lnKey] = ln.(*unixListener)
}
// lightly wrap the listener so that when it is closed,
// we can decrement the usage pool counter
return deleteListener{ln, lnKey}, err
}
// reusePort sets SO_REUSEPORT. Ineffective for unix sockets.
func reusePort(network, address string, conn syscall.RawConn) error {
if IsUnixNetwork(network) {
return nil
}
return conn.Control(func(descriptor uintptr) {
if err := unix.SetsockoptInt(int(descriptor), unix.SOL_SOCKET, unixSOREUSEPORT, 1); err != nil {
Log().Error("setting SO_REUSEPORT",
zap.String("network", network),
zap.String("address", address),
zap.Uintptr("descriptor", descriptor),
zap.Error(err))
}
})
}
type unixListener struct {
*net.UnixListener
mapKey string
count *int32 // accessed atomically
}
func (uln *unixListener) Close() error {
newCount := atomic.AddInt32(uln.count, -1)
if newCount == 0 {
defer func() {
addr := uln.Addr().String()
unixSocketsMu.Lock()
delete(unixSockets, uln.mapKey)
unixSocketsMu.Unlock()
_ = syscall.Unlink(addr)
}()
}
return uln.UnixListener.Close()
}
// deleteListener is a type that simply deletes itself
// from the listenerPool when it closes. It is used
// solely for the purpose of reference counting (i.e.
// counting how many configs are using a given socket).
type deleteListener struct {
net.Listener
lnKey string
}
func (dl deleteListener) Close() error {
_, _ = listenerPool.Delete(dl.lnKey)
return dl.Listener.Close()
}
|
