HTTP cache rework and enable caching for storage assets (#13569)

This enabled HTTP time-based cache for storage assets, primarily
avatars. I have not observed If-Modified-Since from browsers during
tests but I guess it's good to support regardless.

It introduces a new generic httpcache module that can handle both
time-based and etag-based caching.

Additionally, manifest.json and robots.txt are now also cachable.
release/v1.15
silverwind 2020-11-17 23:44:52 +01:00 committed by GitHub
parent 9ec5e6c40b
commit 0615b668dc
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 91 additions and 36 deletions

View File

@ -389,7 +389,7 @@ GRACEFUL_HAMMER_TIME = 60s
; Allows the setting of a startup timeout and waithint for Windows as SVC service ; Allows the setting of a startup timeout and waithint for Windows as SVC service
; 0 disables this. ; 0 disables this.
STARTUP_TIMEOUT = 0 STARTUP_TIMEOUT = 0
; Static resources, includes resources on custom/, public/ and all uploaded avatars web browser cache time, default is 6h ; Static resources, includes resources on custom/, public/ and all uploaded avatars web browser cache time. Note that this cache is disabled when RUN_MODE is "dev". Default is 6h
STATIC_CACHE_TIME = 6h STATIC_CACHE_TIME = 6h
; Define allowed algorithms and their minimum key length (use -1 to disable a type) ; Define allowed algorithms and their minimum key length (use -1 to disable a type)

View File

@ -262,7 +262,7 @@ Values containing `#` or `;` must be quoted using `` ` `` or `"""`.
- `KEY_FILE`: **https/key.pem**: Key file path used for HTTPS. From 1.11 paths are relative to `CUSTOM_PATH`. - `KEY_FILE`: **https/key.pem**: Key file path used for HTTPS. From 1.11 paths are relative to `CUSTOM_PATH`.
- `STATIC_ROOT_PATH`: **./**: Upper level of template and static files path. - `STATIC_ROOT_PATH`: **./**: Upper level of template and static files path.
- `APP_DATA_PATH`: **data** (**/data/gitea** on docker): Default path for application data. - `APP_DATA_PATH`: **data** (**/data/gitea** on docker): Default path for application data.
- `STATIC_CACHE_TIME`: **6h**: Web browser cache time for static resources on `custom/`, `public/` and all uploaded avatars. - `STATIC_CACHE_TIME`: **6h**: Web browser cache time for static resources on `custom/`, `public/` and all uploaded avatars. Note that this cache is disabled when `RUN_MODE` is "dev".
- `ENABLE_GZIP`: **false**: Enables application-level GZIP support. - `ENABLE_GZIP`: **false**: Enables application-level GZIP support.
- `ENABLE_PPROF`: **false**: Application profiling (memory and cpu). For "web" command it listens on localhost:6060. For "serv" command it dumps to disk at `PPROF_DATA_PATH` as `(cpuprofile|memprofile)_<username>_<temporary id>` - `ENABLE_PPROF`: **false**: Application profiling (memory and cpu). For "web" command it listens on localhost:6060. For "serv" command it dumps to disk at `PPROF_DATA_PATH` as `(cpuprofile|memprofile)_<username>_<temporary id>`
- `PPROF_DATA_PATH`: **data/tmp/pprof**: `PPROF_DATA_PATH`, use an absolute path when you start gitea as service - `PPROF_DATA_PATH`: **data/tmp/pprof**: `PPROF_DATA_PATH`, use an absolute path when you start gitea as service

View File

@ -11,6 +11,7 @@ import (
"os" "os"
"runtime" "runtime"
"strings" "strings"
"time"
"code.gitea.io/gitea/cmd" "code.gitea.io/gitea/cmd"
"code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/log"
@ -40,6 +41,7 @@ var (
func init() { func init() {
setting.AppVer = Version setting.AppVer = Version
setting.AppBuiltWith = formatBuiltWith() setting.AppBuiltWith = formatBuiltWith()
setting.AppStartTime = time.Now().UTC()
// Grab the original help templates // Grab the original help templates
originalAppHelpTemplate = cli.AppHelpTemplate originalAppHelpTemplate = cli.AppHelpTemplate

View File

@ -0,0 +1,59 @@
// Copyright 2020 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package httpcache
import (
"encoding/base64"
"fmt"
"net/http"
"os"
"strconv"
"time"
"code.gitea.io/gitea/modules/setting"
)
// GetCacheControl returns a suitable "Cache-Control" header value
func GetCacheControl() string {
if setting.RunMode == "dev" {
return "no-store"
}
return "private, max-age=" + strconv.FormatInt(int64(setting.StaticCacheTime.Seconds()), 10)
}
// generateETag generates an ETag based on size, filename and file modification time
func generateETag(fi os.FileInfo) string {
etag := fmt.Sprint(fi.Size()) + fi.Name() + fi.ModTime().UTC().Format(http.TimeFormat)
return base64.StdEncoding.EncodeToString([]byte(etag))
}
// HandleTimeCache handles time-based caching for a HTTP request
func HandleTimeCache(req *http.Request, w http.ResponseWriter, fi os.FileInfo) (handled bool) {
ifModifiedSince := req.Header.Get("If-Modified-Since")
if ifModifiedSince != "" {
t, err := time.Parse(http.TimeFormat, ifModifiedSince)
if err == nil && fi.ModTime().Unix() <= t.Unix() {
w.WriteHeader(http.StatusNotModified)
return true
}
}
w.Header().Set("Cache-Control", GetCacheControl())
w.Header().Set("Last-Modified", fi.ModTime().Format(http.TimeFormat))
return false
}
// HandleEtagCache handles ETag-based caching for a HTTP request
func HandleEtagCache(req *http.Request, w http.ResponseWriter, fi os.FileInfo) (handled bool) {
etag := generateETag(fi)
if req.Header.Get("If-None-Match") == etag {
w.WriteHeader(http.StatusNotModified)
return true
}
w.Header().Set("Cache-Control", GetCacheControl())
w.Header().Set("ETag", etag)
return false
}

View File

@ -5,15 +5,13 @@
package public package public
import ( import (
"encoding/base64"
"fmt"
"log" "log"
"net/http" "net/http"
"path" "path"
"path/filepath" "path/filepath"
"strings" "strings"
"time"
"code.gitea.io/gitea/modules/httpcache"
"code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/setting"
) )
@ -22,11 +20,8 @@ type Options struct {
Directory string Directory string
IndexFile string IndexFile string
SkipLogging bool SkipLogging bool
// if set to true, will enable caching. Expires header will also be set to FileSystem http.FileSystem
// expire after the defined time. Prefix string
ExpiresAfter time.Duration
FileSystem http.FileSystem
Prefix string
} }
// KnownPublicEntries list all direct children in the `public` directory // KnownPublicEntries list all direct children in the `public` directory
@ -158,23 +153,10 @@ func (opts *Options) handle(w http.ResponseWriter, req *http.Request, opt *Optio
log.Println("[Static] Serving " + file) log.Println("[Static] Serving " + file)
} }
// Add an Expires header to the static content if httpcache.HandleEtagCache(req, w, fi) {
if opt.ExpiresAfter > 0 { return true
w.Header().Set("Expires", time.Now().Add(opt.ExpiresAfter).UTC().Format(http.TimeFormat))
tag := GenerateETag(fmt.Sprint(fi.Size()), fi.Name(), fi.ModTime().UTC().Format(http.TimeFormat))
w.Header().Set("ETag", tag)
if req.Header.Get("If-None-Match") == tag {
w.WriteHeader(304)
return true
}
} }
http.ServeContent(w, req, file, fi.ModTime(), f) http.ServeContent(w, req, file, fi.ModTime(), f)
return true return true
} }
// GenerateETag generates an ETag based on size, filename and file modification time
func GenerateETag(fileSize, fileName, modTime string) string {
etag := fileSize + fileName + modTime
return base64.StdEncoding.EncodeToString([]byte(etag))
}

View File

@ -67,6 +67,7 @@ var (
// AppVer settings // AppVer settings
AppVer string AppVer string
AppBuiltWith string AppBuiltWith string
AppStartTime time.Time
AppName string AppName string
AppURL string AppURL string
AppSubURL string AppSubURL string
@ -362,6 +363,7 @@ var (
PIDFile = "/run/gitea.pid" PIDFile = "/run/gitea.pid"
WritePIDFile bool WritePIDFile bool
ProdMode bool ProdMode bool
RunMode string
RunUser string RunUser string
IsWindows bool IsWindows bool
HasRobotsTxt bool HasRobotsTxt bool
@ -837,6 +839,7 @@ func NewContext() {
} }
RunUser = Cfg.Section("").Key("RUN_USER").MustString(user.CurrentUsername()) RunUser = Cfg.Section("").Key("RUN_USER").MustString(user.CurrentUsername())
RunMode = Cfg.Section("").Key("RUN_MODE").MustString("dev")
// Does not check run user when the install lock is off. // Does not check run user when the install lock is off.
if InstallLock { if InstallLock {
currentUser, match := IsRunUserMatchCurrentUser(RunUser) currentUser, match := IsRunUserMatchCurrentUser(RunUser)

View File

@ -16,6 +16,7 @@ import (
"text/template" "text/template"
"time" "time"
"code.gitea.io/gitea/modules/httpcache"
"code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/metrics" "code.gitea.io/gitea/modules/metrics"
"code.gitea.io/gitea/modules/public" "code.gitea.io/gitea/modules/public"
@ -162,6 +163,12 @@ func storageHandler(storageSetting setting.Storage, prefix string, objStore stor
rPath := strings.TrimPrefix(req.RequestURI, "/"+prefix) rPath := strings.TrimPrefix(req.RequestURI, "/"+prefix)
rPath = strings.TrimPrefix(rPath, "/") rPath = strings.TrimPrefix(rPath, "/")
fi, err := objStore.Stat(rPath)
if err == nil && httpcache.HandleTimeCache(req, w, fi) {
return
}
//If we have matched and access to release or issue //If we have matched and access to release or issue
fr, err := objStore.Open(rPath) fr, err := objStore.Open(rPath)
if err != nil { if err != nil {
@ -200,21 +207,15 @@ func NewChi() chi.Router {
setupAccessLogger(c) setupAccessLogger(c)
} }
if setting.ProdMode {
log.Warn("ProdMode ignored")
}
c.Use(public.Custom( c.Use(public.Custom(
&public.Options{ &public.Options{
SkipLogging: setting.DisableRouterLog, SkipLogging: setting.DisableRouterLog,
ExpiresAfter: time.Hour * 6,
}, },
)) ))
c.Use(public.Static( c.Use(public.Static(
&public.Options{ &public.Options{
Directory: path.Join(setting.StaticRootPath, "public"), Directory: path.Join(setting.StaticRootPath, "public"),
SkipLogging: setting.DisableRouterLog, SkipLogging: setting.DisableRouterLog,
ExpiresAfter: time.Hour * 6,
}, },
)) ))
@ -247,10 +248,14 @@ func NormalRoutes() http.Handler {
w.WriteHeader(http.StatusOK) w.WriteHeader(http.StatusOK)
}) })
// robots.txt
if setting.HasRobotsTxt { if setting.HasRobotsTxt {
r.Get("/robots.txt", func(w http.ResponseWriter, req *http.Request) { r.Get("/robots.txt", func(w http.ResponseWriter, req *http.Request) {
http.ServeFile(w, req, path.Join(setting.CustomPath, "robots.txt")) filePath := path.Join(setting.CustomPath, "robots.txt")
fi, err := os.Stat(filePath)
if err == nil && httpcache.HandleTimeCache(req, w, fi) {
return
}
http.ServeFile(w, req, filePath)
}) })
} }

View File

@ -6,10 +6,12 @@ package routes
import ( import (
"encoding/gob" "encoding/gob"
"net/http"
"code.gitea.io/gitea/models" "code.gitea.io/gitea/models"
"code.gitea.io/gitea/modules/auth" "code.gitea.io/gitea/modules/auth"
"code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/context"
"code.gitea.io/gitea/modules/httpcache"
"code.gitea.io/gitea/modules/lfs" "code.gitea.io/gitea/modules/lfs"
"code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/options" "code.gitea.io/gitea/modules/options"
@ -977,6 +979,8 @@ func RegisterMacaronRoutes(m *macaron.Macaron) {
// Progressive Web App // Progressive Web App
m.Get("/manifest.json", templates.JSONRenderer(), func(ctx *context.Context) { m.Get("/manifest.json", templates.JSONRenderer(), func(ctx *context.Context) {
ctx.Resp.Header().Set("Cache-Control", httpcache.GetCacheControl())
ctx.Resp.Header().Set("Last-Modified", setting.AppStartTime.Format(http.TimeFormat))
ctx.HTML(200, "pwa/manifest_json") ctx.HTML(200, "pwa/manifest_json")
}) })