1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
|
// Copyright 2016 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package data
import (
"bytes"
"fmt"
"io"
"net/http"
"net/url"
"path/filepath"
"time"
"github.com/gohugoio/hugo/cache/filecache"
"github.com/gohugoio/hugo/common/hashing"
"github.com/spf13/afero"
)
var (
resSleep = time.Second * 2 // if JSON decoding failed sleep for n seconds before retrying
resRetries = 1 // number of retries to load the JSON from URL
)
// getRemote loads the content of a remote file. This method is thread safe.
func (ns *Namespace) getRemote(cache *filecache.Cache, unmarshal func([]byte) (bool, error), req *http.Request) error {
url := req.URL.String()
if err := ns.deps.ExecHelper.Sec().CheckAllowedHTTPURL(url); err != nil {
return err
}
if err := ns.deps.ExecHelper.Sec().CheckAllowedHTTPMethod("GET"); err != nil {
return err
}
var headers bytes.Buffer
req.Header.Write(&headers)
id := hashing.MD5FromStringHexEncoded(url + headers.String())
var handled bool
var retry bool
_, b, err := cache.GetOrCreateBytes(id, func() ([]byte, error) {
var err error
handled = true
for i := 0; i <= resRetries; i++ {
ns.deps.Log.Infof("Downloading: %s ...", url)
var res *http.Response
res, err = ns.client.Do(req)
if err != nil {
return nil, err
}
var b []byte
b, err = io.ReadAll(res.Body)
if err != nil {
return nil, err
}
res.Body.Close()
if isHTTPError(res) {
return nil, fmt.Errorf("failed to retrieve remote file: %s, body: %q", http.StatusText(res.StatusCode), b)
}
retry, err = unmarshal(b)
if err == nil {
// Return it so it can be cached.
return b, nil
}
if !retry {
return nil, err
}
ns.deps.Log.Infof("Cannot read remote resource %s: %s", url, err)
ns.deps.Log.Infof("Retry #%d for %s and sleeping for %s", i+1, url, resSleep)
time.Sleep(resSleep)
}
return nil, err
})
if !handled {
// This is cached content and should be correct.
_, err = unmarshal(b)
}
return err
}
// getLocal loads the content of a local file
func getLocal(workingDir, url string, fs afero.Fs) ([]byte, error) {
filename := filepath.Join(workingDir, url)
return afero.ReadFile(fs, filename)
}
// getResource loads the content of a local or remote file and returns its content and the
// cache ID used, if relevant.
func (ns *Namespace) getResource(cache *filecache.Cache, unmarshal func(b []byte) (bool, error), req *http.Request) error {
switch req.URL.Scheme {
case "":
url, err := url.QueryUnescape(req.URL.String())
if err != nil {
return err
}
b, err := getLocal(ns.deps.Conf.BaseConfig().WorkingDir, url, ns.deps.Fs.Source)
if err != nil {
return err
}
_, err = unmarshal(b)
return err
default:
return ns.getRemote(cache, unmarshal, req)
}
}
func isHTTPError(res *http.Response) bool {
return res.StatusCode < 200 || res.StatusCode > 299
}
|
