proxy.go

v1.3.3

package server

import (
	"context"
	"errors"
	"fmt"
	"io"
	"log"
	"net/http"
	"strings"

	"golang.org/x/mod/module"

	"go.bigb.es/curator/internal/build"
	"go.bigb.es/curator/internal/git"
	"go.bigb.es/curator/internal/metrics"
	"go.bigb.es/curator/internal/storage"
)

func (s *Server) serveProxy(w http.ResponseWriter, r *http.Request, escapedModPath, file string) {
	// Unescape the module path (e.g., "!azure" → "Azure").
	modPath, err := module.UnescapePath(escapedModPath)
	if err != nil {
		log.Printf("proxy: invalid module path %q: %v", escapedModPath, err)
		http.NotFound(w, r)
		return
	}

	localName := modPath
	if strings.Contains(modPath, "/") {
		if after, ok := strings.CutPrefix(modPath, s.Cfg.Host+"/"); ok {
			localName = after
		}
	}

	if mod, ok := s.Resolver.ResolveModule(localName); ok {
		_ = mod // used below
		if !s.CanAccessModule(localName, r) {
			http.NotFound(w, r)
			return
		}

		s.serveLocalModule(w, r, localName, file)
		return
	}

	s.serveFallback(w, r, escapedModPath, file)
}

func (s *Server) serveLocalModule(w http.ResponseWriter, r *http.Request, modName, file string) {
	mod, _ := s.Resolver.ResolveModule(modName)
	modulePath := s.Cfg.Host + "/" + modName

	// For version-specific artifacts, try S3 cache first.
	if s.Store != nil && file != "list" {
		key := StorageKey(modName, file)
		data, ct, err := s.Store.Get(r.Context(), key)
		if err == nil {
			metrics.ProxyCacheHitsTotal.Inc()
			w.Header().Set("Content-Type", ct)
			w.Write(data)
			return
		}

		metrics.ProxyCacheMissesTotal.Inc()
		if !errors.Is(err, storage.ErrNotFound) {
			log.Printf("s3 get %s: %v", key, err)
		}
	}

	repoPath, err := s.Git.CloneOrFetch(modName, mod.Repo, s.authForModule(mod))
	if err != nil {
		log.Printf("git error for %s: %v", modName, err)
		http.Error(w, "internal error", http.StatusInternalServerError)
		return
	}

	switch {
	case file == "list":
		s.serveList(w, repoPath, modulePath)
	case strings.HasSuffix(file, ".info"):
		escapedVer := strings.TrimSuffix(file, ".info")
		query, err := module.UnescapeVersion(escapedVer)
		if err != nil {
			log.Printf("proxy: invalid version %q: %v", escapedVer, err)
			http.NotFound(w, r)
			return
		}
		rv, err := git.ResolveVersion(repoPath, query)
		if err != nil {
			log.Printf("resolve %s: %v", query, err)
			http.NotFound(w, r)
			return
		}
		cacheFile := rv.Version + ".info"
		s.serveVersioned(w, r, modName, cacheFile, func() ([]byte, string, error) {
			return build.Info(repoPath, rv, modulePath)
		})
	case strings.HasSuffix(file, ".mod"):
		escapedVer := strings.TrimSuffix(file, ".mod")
		query, err := module.UnescapeVersion(escapedVer)
		if err != nil {
			log.Printf("proxy: invalid version %q: %v", escapedVer, err)
			http.NotFound(w, r)
			return
		}
		rv, err := git.ResolveVersion(repoPath, query)
		if err != nil {
			log.Printf("resolve %s: %v", query, err)
			http.NotFound(w, r)
			return
		}
		cacheFile := rv.Version + ".mod"
		s.serveVersioned(w, r, modName, cacheFile, func() ([]byte, string, error) {
			return build.Mod(repoPath, rv, modulePath)
		})
	case strings.HasSuffix(file, ".zip"):
		escapedVer := strings.TrimSuffix(file, ".zip")
		query, err := module.UnescapeVersion(escapedVer)
		if err != nil {
			log.Printf("proxy: invalid version %q: %v", escapedVer, err)
			http.NotFound(w, r)
			return
		}
		rv, err := git.ResolveVersion(repoPath, query)
		if err != nil {
			log.Printf("resolve %s: %v", query, err)
			http.NotFound(w, r)
			return
		}
		cacheFile := rv.Version + ".zip"
		s.serveVersioned(w, r, modName, cacheFile, func() ([]byte, string, error) {
			return build.Zip(repoPath, rv, modulePath)
		})
	default:
		http.NotFound(w, r)
	}
}

// serveLatest handles GET $base/$module/@latest requests.
func (s *Server) serveLatest(w http.ResponseWriter, r *http.Request, escapedModPath string) {
	modPath, err := module.UnescapePath(escapedModPath)
	if err != nil {
		log.Printf("proxy: invalid module path %q: %v", escapedModPath, err)
		http.NotFound(w, r)
		return
	}

	localName := modPath
	if strings.Contains(modPath, "/") {
		if after, ok := strings.CutPrefix(modPath, s.Cfg.Host+"/"); ok {
			localName = after
		}
	}

	mod, ok := s.Resolver.ResolveModule(localName)
	if !ok {
		// Try fallback for @latest.
		s.serveFallbackLatest(w, r, escapedModPath)
		return
	}

	if !s.CanAccessModule(localName, r) {
		http.NotFound(w, r)
		return
	}

	modulePath := s.Cfg.Host + "/" + localName

	repoPath, err := s.Git.CloneOrFetch(localName, mod.Repo, s.authForModule(mod))
	if err != nil {
		log.Printf("git error for %s: %v", localName, err)
		http.Error(w, "internal error", http.StatusInternalServerError)
		return
	}

	allTags, err := git.ListTags(repoPath)
	if err != nil || len(allTags) == 0 {
		http.NotFound(w, r)
		return
	}

	tags := git.FilterTagsByMajor(allTags, modulePath)
	if len(tags) == 0 {
		http.NotFound(w, r)
		return
	}

	latest := git.LatestRelease(tags)
	rv, err := git.ResolveVersion(repoPath, latest)
	if err != nil {
		log.Printf("resolve latest %s: %v", latest, err)
		http.NotFound(w, r)
		return
	}

	data, ct, err := build.Info(repoPath, rv, modulePath)
	if err != nil {
		log.Printf("build info for %s@latest: %v", localName, err)
		http.Error(w, "internal error", http.StatusInternalServerError)
		return
	}

	w.Header().Set("Content-Type", ct)
	w.Write(data)
}

func (s *Server) serveFallback(w http.ResponseWriter, r *http.Request, modName, file string) {
	mode := s.Cfg.FallbackMode()
	switch mode {
	case "redirect":
		http.Redirect(w, r, s.upstreamURL(modName, file), http.StatusFound)
	case "sync":
		s.fetchAndCache(w, r, modName, file, true)
	case "async":
		s.fetchAndCache(w, r, modName, file, false)
	default:
		http.NotFound(w, r)
	}
}

func (s *Server) serveFallbackLatest(w http.ResponseWriter, r *http.Request, modName string) {
	mode := s.Cfg.FallbackMode()
	upURL := strings.TrimRight(s.Cfg.Fallback.Upstream, "/") + "/" + modName + "/@latest"

	switch mode {
	case "redirect":
		http.Redirect(w, r, upURL, http.StatusFound)
	case "sync", "async":
		req, err := http.NewRequestWithContext(r.Context(), http.MethodGet, upURL, nil)
		if err != nil {
			http.Error(w, "internal error", http.StatusInternalServerError)
			return
		}
		resp, err := s.HTTPClient.Do(req)
		if err != nil {
			http.Error(w, "upstream error", http.StatusBadGateway)
			return
		}
		defer resp.Body.Close()
		w.WriteHeader(resp.StatusCode)
		io.Copy(w, resp.Body)
	default:
		http.NotFound(w, r)
	}
}

func (s *Server) upstreamURL(modName, file string) string {
	return strings.TrimRight(s.Cfg.Fallback.Upstream, "/") + "/" + modName + "/@v/" + file
}

// fetchAndCache fetches from upstream and optionally caches.
// When syncCache is true (sync mode), the cache write completes before responding.
// When false (async mode), the cache write runs in a background goroutine.
func (s *Server) fetchAndCache(w http.ResponseWriter, r *http.Request, modName, file string, syncCache bool) {
	// Try cache first for both sync and async modes.
	if s.Store != nil {
		key := StorageKey(modName, file)
		data, ct, err := s.Store.Get(r.Context(), key)
		if err == nil {
			metrics.ProxyCacheHitsTotal.Inc()
			w.Header().Set("Content-Type", ct)
			w.Write(data)
			return
		}

		if !errors.Is(err, storage.ErrNotFound) {
			log.Printf("storage get %s: %v", key, err)
		}
	}

	upURL := s.upstreamURL(modName, file)

	req, err := http.NewRequestWithContext(r.Context(), http.MethodGet, upURL, nil)
	if err != nil {
		log.Printf("upstream request: %v", err)
		http.Error(w, "internal error", http.StatusInternalServerError)
		return
	}

	resp, err := s.HTTPClient.Do(req)
	if err != nil {
		log.Printf("upstream fetch %s: %v", upURL, err)
		http.Error(w, "upstream error", http.StatusBadGateway)
		return
	}
	defer resp.Body.Close()

	if resp.StatusCode != http.StatusOK {
		w.WriteHeader(resp.StatusCode)
		io.Copy(w, resp.Body)
		return
	}

	body, err := io.ReadAll(resp.Body)
	if err != nil {
		log.Printf("upstream read %s: %v", upURL, err)
		http.Error(w, "upstream error", http.StatusBadGateway)
		return
	}

	ct := resp.Header.Get("Content-Type")

	if s.Store != nil {
		key := StorageKey(modName, file)
		if syncCache {
			if err := s.Store.Put(context.Background(), key, body, ct); err != nil {
				log.Printf("storage put %s: %v", key, err)
			}
		} else {
			go func() {
				if err := s.Store.Put(context.Background(), key, body, ct); err != nil {
					log.Printf("storage put %s: %v", key, err)
				}
			}()
		}
	}

	if ct != "" {
		w.Header().Set("Content-Type", ct)
	}

	w.Write(body)
}

func (s *Server) serveVersioned(w http.ResponseWriter, r *http.Request, modName, file string, buildFn func() ([]byte, string, error)) {
	data, ct, err := buildFn()
	if err != nil {
		log.Printf("build %s/%s: %v", modName, file, err)
		http.NotFound(w, nil)
		return
	}

	if s.Store != nil {
		key := StorageKey(modName, file)
		go func() {
			if err := s.Store.Put(context.Background(), key, data, ct); err != nil {
				log.Printf("s3 put %s: %v", key, err)
			}
		}()
	}

	w.Header().Set("Content-Type", ct)
	w.Write(data)
}

func (s *Server) serveList(w http.ResponseWriter, repoPath, modulePath string) {
	tags, err := git.ListTags(repoPath)
	if err != nil {
		log.Printf("list tags: %v", err)
		http.Error(w, "internal error", http.StatusInternalServerError)
		return
	}

	tags = git.FilterTagsByMajor(tags, modulePath)

	w.Header().Set("Content-Type", "text/plain; charset=utf-8")
	for _, tag := range tags {
		_, t, err := git.CommitInfo(repoPath, tag)
		if err != nil {
			// Fall back to version only (no timestamp).
			fmt.Fprintln(w, tag)
			continue
		}
		fmt.Fprintf(w, "%s %s\n", tag, t.UTC().Format("2006-01-02T15:04:05Z"))
	}
}


1	package server
2
3	import (
4	"context"
5	"errors"
6	"fmt"
7	"io"
8	"log"
9	"net/http"
10	"strings"
11
12	"golang.org/x/mod/module"
13
14	"go.bigb.es/curator/internal/build"
15	"go.bigb.es/curator/internal/git"
16	"go.bigb.es/curator/internal/metrics"
17	"go.bigb.es/curator/internal/storage"
18	)
19
20	func (s Server) serveProxy(w http.ResponseWriter, r http.Request, escapedModPath, file string) {
21	// Unescape the module path (e.g., "!azure" → "Azure").
22	modPath, err := module.UnescapePath(escapedModPath)
23	if err != nil {
24	log.Printf("proxy: invalid module path %q: %v", escapedModPath, err)
25	http.NotFound(w, r)
26	return
27	}
28
29	localName := modPath
30	if strings.Contains(modPath, "/") {
31	if after, ok := strings.CutPrefix(modPath, s.Cfg.Host+"/"); ok {
32	localName = after
33	}
34	}
35
36	if mod, ok := s.Resolver.ResolveModule(localName); ok {
37	_ = mod // used below
38	if !s.CanAccessModule(localName, r) {
39	http.NotFound(w, r)
40	return
41	}
42
43	s.serveLocalModule(w, r, localName, file)
44	return
45	}
46
47	s.serveFallback(w, r, escapedModPath, file)
48	}
49
50	func (s Server) serveLocalModule(w http.ResponseWriter, r http.Request, modName, file string) {
51	mod, _ := s.Resolver.ResolveModule(modName)
52	modulePath := s.Cfg.Host + "/" + modName
53
54	// For version-specific artifacts, try S3 cache first.
55	if s.Store != nil && file != "list" {
56	key := StorageKey(modName, file)
57	data, ct, err := s.Store.Get(r.Context(), key)
58	if err == nil {
59	metrics.ProxyCacheHitsTotal.Inc()
60	w.Header().Set("Content-Type", ct)
61	w.Write(data)
62	return
63	}
64
65	metrics.ProxyCacheMissesTotal.Inc()
66	if !errors.Is(err, storage.ErrNotFound) {
67	log.Printf("s3 get %s: %v", key, err)
68	}
69	}
70
71	repoPath, err := s.Git.CloneOrFetch(modName, mod.Repo, s.authForModule(mod))
72	if err != nil {
73	log.Printf("git error for %s: %v", modName, err)
74	http.Error(w, "internal error", http.StatusInternalServerError)
75	return
76	}
77
78	switch {
79	case file == "list":
80	s.serveList(w, repoPath, modulePath)
81	case strings.HasSuffix(file, ".info"):
82	escapedVer := strings.TrimSuffix(file, ".info")
83	query, err := module.UnescapeVersion(escapedVer)
84	if err != nil {
85	log.Printf("proxy: invalid version %q: %v", escapedVer, err)
86	http.NotFound(w, r)
87	return
88	}
89	rv, err := git.ResolveVersion(repoPath, query)
90	if err != nil {
91	log.Printf("resolve %s: %v", query, err)
92	http.NotFound(w, r)
93	return
94	}
95	cacheFile := rv.Version + ".info"
96	s.serveVersioned(w, r, modName, cacheFile, func() ([]byte, string, error) {
97	return build.Info(repoPath, rv, modulePath)
98	})
99	case strings.HasSuffix(file, ".mod"):
100	escapedVer := strings.TrimSuffix(file, ".mod")
101	query, err := module.UnescapeVersion(escapedVer)
102	if err != nil {
103	log.Printf("proxy: invalid version %q: %v", escapedVer, err)
104	http.NotFound(w, r)
105	return
106	}
107	rv, err := git.ResolveVersion(repoPath, query)
108	if err != nil {
109	log.Printf("resolve %s: %v", query, err)
110	http.NotFound(w, r)
111	return
112	}
113	cacheFile := rv.Version + ".mod"
114	s.serveVersioned(w, r, modName, cacheFile, func() ([]byte, string, error) {
115	return build.Mod(repoPath, rv, modulePath)
116	})
117	case strings.HasSuffix(file, ".zip"):
118	escapedVer := strings.TrimSuffix(file, ".zip")
119	query, err := module.UnescapeVersion(escapedVer)
120	if err != nil {
121	log.Printf("proxy: invalid version %q: %v", escapedVer, err)
122	http.NotFound(w, r)
123	return
124	}
125	rv, err := git.ResolveVersion(repoPath, query)
126	if err != nil {
127	log.Printf("resolve %s: %v", query, err)
128	http.NotFound(w, r)
129	return
130	}
131	cacheFile := rv.Version + ".zip"
132	s.serveVersioned(w, r, modName, cacheFile, func() ([]byte, string, error) {
133	return build.Zip(repoPath, rv, modulePath)
134	})
135	default:
136	http.NotFound(w, r)
137	}
138	}
139
140	// serveLatest handles GET $base/$module/@latest requests.
141	func (s Server) serveLatest(w http.ResponseWriter, r http.Request, escapedModPath string) {
142	modPath, err := module.UnescapePath(escapedModPath)
143	if err != nil {
144	log.Printf("proxy: invalid module path %q: %v", escapedModPath, err)
145	http.NotFound(w, r)
146	return
147	}
148
149	localName := modPath
150	if strings.Contains(modPath, "/") {
151	if after, ok := strings.CutPrefix(modPath, s.Cfg.Host+"/"); ok {
152	localName = after
153	}
154	}
155
156	mod, ok := s.Resolver.ResolveModule(localName)
157	if !ok {
158	// Try fallback for @latest.
159	s.serveFallbackLatest(w, r, escapedModPath)
160	return
161	}
162
163	if !s.CanAccessModule(localName, r) {
164	http.NotFound(w, r)
165	return
166	}
167
168	modulePath := s.Cfg.Host + "/" + localName
169
170	repoPath, err := s.Git.CloneOrFetch(localName, mod.Repo, s.authForModule(mod))
171	if err != nil {
172	log.Printf("git error for %s: %v", localName, err)
173	http.Error(w, "internal error", http.StatusInternalServerError)
174	return
175	}
176
177	allTags, err := git.ListTags(repoPath)
178	if err != nil \|\| len(allTags) == 0 {
179	http.NotFound(w, r)
180	return
181	}
182
183	tags := git.FilterTagsByMajor(allTags, modulePath)
184	if len(tags) == 0 {
185	http.NotFound(w, r)
186	return
187	}
188
189	latest := git.LatestRelease(tags)
190	rv, err := git.ResolveVersion(repoPath, latest)
191	if err != nil {
192	log.Printf("resolve latest %s: %v", latest, err)
193	http.NotFound(w, r)
194	return
195	}
196
197	data, ct, err := build.Info(repoPath, rv, modulePath)
198	if err != nil {
199	log.Printf("build info for %s@latest: %v", localName, err)
200	http.Error(w, "internal error", http.StatusInternalServerError)
201	return
202	}
203
204	w.Header().Set("Content-Type", ct)
205	w.Write(data)
206	}
207
208	func (s Server) serveFallback(w http.ResponseWriter, r http.Request, modName, file string) {
209	mode := s.Cfg.FallbackMode()
210	switch mode {
211	case "redirect":
212	http.Redirect(w, r, s.upstreamURL(modName, file), http.StatusFound)
213	case "sync":
214	s.fetchAndCache(w, r, modName, file, true)
215	case "async":
216	s.fetchAndCache(w, r, modName, file, false)
217	default:
218	http.NotFound(w, r)
219	}
220	}
221
222	func (s Server) serveFallbackLatest(w http.ResponseWriter, r http.Request, modName string) {
223	mode := s.Cfg.FallbackMode()
224	upURL := strings.TrimRight(s.Cfg.Fallback.Upstream, "/") + "/" + modName + "/@latest"
225
226	switch mode {
227	case "redirect":
228	http.Redirect(w, r, upURL, http.StatusFound)
229	case "sync", "async":
230	req, err := http.NewRequestWithContext(r.Context(), http.MethodGet, upURL, nil)
231	if err != nil {
232	http.Error(w, "internal error", http.StatusInternalServerError)
233	return
234	}
235	resp, err := s.HTTPClient.Do(req)
236	if err != nil {
237	http.Error(w, "upstream error", http.StatusBadGateway)
238	return
239	}
240	defer resp.Body.Close()
241	w.WriteHeader(resp.StatusCode)
242	io.Copy(w, resp.Body)
243	default:
244	http.NotFound(w, r)
245	}
246	}
247
248	func (s *Server) upstreamURL(modName, file string) string {
249	return strings.TrimRight(s.Cfg.Fallback.Upstream, "/") + "/" + modName + "/@v/" + file
250	}
251
252	// fetchAndCache fetches from upstream and optionally caches.
253	// When syncCache is true (sync mode), the cache write completes before responding.
254	// When false (async mode), the cache write runs in a background goroutine.
255	func (s Server) fetchAndCache(w http.ResponseWriter, r http.Request, modName, file string, syncCache bool) {
256	// Try cache first for both sync and async modes.
257	if s.Store != nil {
258	key := StorageKey(modName, file)
259	data, ct, err := s.Store.Get(r.Context(), key)
260	if err == nil {
261	metrics.ProxyCacheHitsTotal.Inc()
262	w.Header().Set("Content-Type", ct)
263	w.Write(data)
264	return
265	}
266
267	if !errors.Is(err, storage.ErrNotFound) {
268	log.Printf("storage get %s: %v", key, err)
269	}
270	}
271
272	upURL := s.upstreamURL(modName, file)
273
274	req, err := http.NewRequestWithContext(r.Context(), http.MethodGet, upURL, nil)
275	if err != nil {
276	log.Printf("upstream request: %v", err)
277	http.Error(w, "internal error", http.StatusInternalServerError)
278	return
279	}
280
281	resp, err := s.HTTPClient.Do(req)
282	if err != nil {
283	log.Printf("upstream fetch %s: %v", upURL, err)
284	http.Error(w, "upstream error", http.StatusBadGateway)
285	return
286	}
287	defer resp.Body.Close()
288
289	if resp.StatusCode != http.StatusOK {
290	w.WriteHeader(resp.StatusCode)
291	io.Copy(w, resp.Body)
292	return
293	}
294
295	body, err := io.ReadAll(resp.Body)
296	if err != nil {
297	log.Printf("upstream read %s: %v", upURL, err)
298	http.Error(w, "upstream error", http.StatusBadGateway)
299	return
300	}
301
302	ct := resp.Header.Get("Content-Type")
303
304	if s.Store != nil {
305	key := StorageKey(modName, file)
306	if syncCache {
307	if err := s.Store.Put(context.Background(), key, body, ct); err != nil {
308	log.Printf("storage put %s: %v", key, err)
309	}
310	} else {
311	go func() {
312	if err := s.Store.Put(context.Background(), key, body, ct); err != nil {
313	log.Printf("storage put %s: %v", key, err)
314	}
315	}()
316	}
317	}
318
319	if ct != "" {
320	w.Header().Set("Content-Type", ct)
321	}
322
323	w.Write(body)
324	}
325
326	func (s Server) serveVersioned(w http.ResponseWriter, r http.Request, modName, file string, buildFn func() ([]byte, string, error)) {
327	data, ct, err := buildFn()
328	if err != nil {
329	log.Printf("build %s/%s: %v", modName, file, err)
330	http.NotFound(w, nil)
331	return
332	}
333
334	if s.Store != nil {
335	key := StorageKey(modName, file)
336	go func() {
337	if err := s.Store.Put(context.Background(), key, data, ct); err != nil {
338	log.Printf("s3 put %s: %v", key, err)
339	}
340	}()
341	}
342
343	w.Header().Set("Content-Type", ct)
344	w.Write(data)
345	}
346
347	func (s *Server) serveList(w http.ResponseWriter, repoPath, modulePath string) {
348	tags, err := git.ListTags(repoPath)
349	if err != nil {
350	log.Printf("list tags: %v", err)
351	http.Error(w, "internal error", http.StatusInternalServerError)
352	return
353	}
354
355	tags = git.FilterTagsByMajor(tags, modulePath)
356
357	w.Header().Set("Content-Type", "text/plain; charset=utf-8")
358	for _, tag := range tags {
359	_, t, err := git.CommitInfo(repoPath, tag)
360	if err != nil {
361	// Fall back to version only (no timestamp).
362	fmt.Fprintln(w, tag)
363	continue
364	}
365	fmt.Fprintf(w, "%s %s\n", tag, t.UTC().Format("2006-01-02T15:04:05Z"))
366	}
367	}
368

proxy.go

Source Files