初始提交: Gitea 项目代码

This commit is contained in:
root
2026-05-30 22:47:36 +08:00
commit f288f76350
6116 changed files with 776822 additions and 0 deletions
+119
View File
@@ -0,0 +1,119 @@
// Copyright 2017 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package cache
import (
"encoding/hex"
"errors"
"fmt"
"strconv"
"time"
"gitea.dev/modules/setting"
_ "gitea.com/go-chi/cache/memcache" //nolint:depguard // memcache plugin for cache, it is required for config "ADAPTER=memcache"
)
var defaultCache StringCache
// Init start cache service
func Init() error {
if defaultCache == nil {
c, err := NewStringCache(setting.CacheService.Cache)
if err != nil {
return err
}
for range 10 {
if err = c.Ping(); err == nil {
break
}
time.Sleep(time.Second)
}
if err != nil {
return err
}
defaultCache = c
}
return nil
}
const (
testCacheKey = "DefaultCache.TestKey"
// SlowCacheThreshold marks cache tests as slow
// set to 30ms per discussion: https://github.com/go-gitea/gitea/issues/33190
// TODO: Replace with metrics histogram
SlowCacheThreshold = 30 * time.Millisecond
)
// Test performs delete, put and get operations on a predefined key
// returns
func Test() (time.Duration, error) {
if defaultCache == nil {
return 0, errors.New("default cache not initialized")
}
testData := hex.EncodeToString(make([]byte, 500))
start := time.Now()
if err := defaultCache.Delete(testCacheKey); err != nil {
return 0, fmt.Errorf("expect cache to delete data based on key if exist but got: %w", err)
}
if err := defaultCache.Put(testCacheKey, testData, 10); err != nil {
return 0, fmt.Errorf("expect cache to store data but got: %w", err)
}
testVal, hit := defaultCache.Get(testCacheKey)
if !hit {
return 0, errors.New("expect cache hit but got none")
}
if testVal != testData {
return 0, errors.New("expect cache to return same value as stored but got other")
}
return time.Since(start), nil
}
// GetCache returns the currently configured cache
func GetCache() StringCache {
return defaultCache
}
// GetString returns the key value from cache with callback when no key exists in cache
func GetString(key string, getFunc func() (string, error)) (string, error) {
if defaultCache == nil || setting.CacheService.TTL == 0 {
return getFunc()
}
cached, exist := defaultCache.Get(key)
if !exist {
value, err := getFunc()
if err != nil {
return value, err
}
return value, defaultCache.Put(key, value, setting.CacheService.TTLSeconds())
}
return cached, nil
}
// GetInt64 returns key value from cache with callback when no key exists in cache
func GetInt64(key string, getFunc func() (int64, error)) (int64, error) {
s, err := GetString(key, func() (string, error) {
v, err := getFunc()
return strconv.FormatInt(v, 10), err
})
if err != nil {
return 0, err
}
if s == "" {
return 0, nil
}
return strconv.ParseInt(s, 10, 64)
}
// Remove key from cache
func Remove(key string) {
if defaultCache == nil {
return
}
_ = defaultCache.Delete(key)
}
+162
View File
@@ -0,0 +1,162 @@
// Copyright 2020 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package cache
import (
"fmt"
"strconv"
"time"
"gitea.dev/modules/graceful"
"gitea.dev/modules/nosql"
"gitea.com/go-chi/cache" //nolint:depguard // we wrap this package here
"github.com/redis/go-redis/v9"
)
// RedisCacher represents a redis cache adapter implementation.
type RedisCacher struct {
c redis.UniversalClient
prefix string
hsetName string
occupyMode bool
}
// toStr convert string/int/int64 interface to string. it's only used by the RedisCacher.Put internally
func toStr(v any) string {
if v == nil {
return ""
}
switch v := v.(type) {
case string:
return v
case []byte:
return string(v)
case int:
return strconv.FormatInt(int64(v), 10)
case int64:
return strconv.FormatInt(v, 10)
default:
return fmt.Sprint(v) // as what the old com.ToStr does in most cases
}
}
// Put puts value (string type) into cache with key and expire time.
// If expired is 0, it lives forever.
func (c *RedisCacher) Put(key string, val any, expire int64) error {
// this function is not well-designed, it only puts string values into cache
key = c.prefix + key
if expire == 0 {
if err := c.c.Set(graceful.GetManager().HammerContext(), key, toStr(val), 0).Err(); err != nil {
return err
}
} else {
dur := time.Duration(expire) * time.Second
if err := c.c.Set(graceful.GetManager().HammerContext(), key, toStr(val), dur).Err(); err != nil {
return err
}
}
if c.occupyMode {
return nil
}
return c.c.HSet(graceful.GetManager().HammerContext(), c.hsetName, key, "0").Err()
}
// Get gets cached value by given key.
func (c *RedisCacher) Get(key string) any {
val, err := c.c.Get(graceful.GetManager().HammerContext(), c.prefix+key).Result()
if err != nil {
return nil
}
return val
}
// Delete deletes cached value by given key.
func (c *RedisCacher) Delete(key string) error {
key = c.prefix + key
if err := c.c.Del(graceful.GetManager().HammerContext(), key).Err(); err != nil {
return err
}
if c.occupyMode {
return nil
}
return c.c.HDel(graceful.GetManager().HammerContext(), c.hsetName, key).Err()
}
// Incr increases cached int-type value by given key as a counter.
func (c *RedisCacher) Incr(key string) error {
if !c.IsExist(key) {
return fmt.Errorf("key '%s' not exist", key)
}
return c.c.Incr(graceful.GetManager().HammerContext(), c.prefix+key).Err()
}
// Decr decreases cached int-type value by given key as a counter.
func (c *RedisCacher) Decr(key string) error {
if !c.IsExist(key) {
return fmt.Errorf("key '%s' not exist", key)
}
return c.c.Decr(graceful.GetManager().HammerContext(), c.prefix+key).Err()
}
// IsExist returns true if cached value exists.
func (c *RedisCacher) IsExist(key string) bool {
if c.c.Exists(graceful.GetManager().HammerContext(), c.prefix+key).Val() == 1 {
return true
}
if !c.occupyMode {
c.c.HDel(graceful.GetManager().HammerContext(), c.hsetName, c.prefix+key)
}
return false
}
// Flush deletes all cached data.
func (c *RedisCacher) Flush() error {
if c.occupyMode {
return c.c.FlushDB(graceful.GetManager().HammerContext()).Err()
}
keys, err := c.c.HKeys(graceful.GetManager().HammerContext(), c.hsetName).Result()
if err != nil {
return err
}
if err = c.c.Del(graceful.GetManager().HammerContext(), keys...).Err(); err != nil {
return err
}
return c.c.Del(graceful.GetManager().HammerContext(), c.hsetName).Err()
}
// StartAndGC starts GC routine based on config string settings.
// AdapterConfig: network=tcp,addr=:6379,password=macaron,db=0,pool_size=100,idle_timeout=180,hset_name=MacaronCache,prefix=cache:
func (c *RedisCacher) StartAndGC(opts cache.Options) error {
c.hsetName = "MacaronCache"
c.occupyMode = opts.OccupyMode
uri := nosql.ToRedisURI(opts.AdapterConfig)
c.c = nosql.GetManager().GetRedisClient(uri.String())
for k, v := range uri.Query() {
switch k {
case "hset_name":
c.hsetName = v[0]
case "prefix":
c.prefix = v[0]
}
}
return c.c.Ping(graceful.GetManager().HammerContext()).Err()
}
// Ping tests if the cache is alive.
func (c *RedisCacher) Ping() error {
return c.c.Ping(graceful.GetManager().HammerContext()).Err()
}
func init() {
cache.Register("redis", &RedisCacher{})
}
+126
View File
@@ -0,0 +1,126 @@
// Copyright 2021 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package cache
import (
"errors"
"testing"
"time"
"gitea.dev/modules/setting"
"github.com/stretchr/testify/assert"
)
func createTestCache() {
defaultCache, _ = NewStringCache(setting.Cache{
Adapter: "memory",
TTL: time.Minute,
})
setting.CacheService.TTL = 24 * time.Hour
}
func TestNewContext(t *testing.T) {
assert.NoError(t, Init())
setting.CacheService.Cache = setting.Cache{Adapter: "redis", Conn: "some random string"}
con, err := NewStringCache(setting.Cache{
Adapter: "rand",
Conn: "false conf",
Interval: 100,
})
assert.Error(t, err)
assert.Nil(t, con)
}
func TestTest(t *testing.T) {
defaultCache = nil
_, err := Test()
assert.Error(t, err)
createTestCache()
elapsed, err := Test()
assert.NoError(t, err)
// mem cache should take from 300ns up to 1ms on modern hardware ...
assert.Positive(t, elapsed)
assert.Less(t, elapsed, SlowCacheThreshold)
}
func TestGetCache(t *testing.T) {
createTestCache()
assert.NotNil(t, GetCache())
}
func TestGetString(t *testing.T) {
createTestCache()
data, err := GetString("key", func() (string, error) {
return "", errors.New("some error")
})
assert.Error(t, err)
assert.Empty(t, data)
data, err = GetString("key", func() (string, error) {
return "", nil
})
assert.NoError(t, err)
assert.Empty(t, data)
data, err = GetString("key", func() (string, error) {
return "some data", nil
})
assert.NoError(t, err)
assert.Empty(t, data)
Remove("key")
data, err = GetString("key", func() (string, error) {
return "some data", nil
})
assert.NoError(t, err)
assert.Equal(t, "some data", data)
data, err = GetString("key", func() (string, error) {
return "", errors.New("some error")
})
assert.NoError(t, err)
assert.Equal(t, "some data", data)
Remove("key")
}
func TestGetInt64(t *testing.T) {
createTestCache()
data, err := GetInt64("key", func() (int64, error) {
return 0, errors.New("some error")
})
assert.Error(t, err)
assert.EqualValues(t, 0, data)
data, err = GetInt64("key", func() (int64, error) {
return 0, nil
})
assert.NoError(t, err)
assert.EqualValues(t, 0, data)
data, err = GetInt64("key", func() (int64, error) {
return 100, nil
})
assert.NoError(t, err)
assert.EqualValues(t, 0, data)
Remove("key")
data, err = GetInt64("key", func() (int64, error) {
return 100, nil
})
assert.NoError(t, err)
assert.EqualValues(t, 100, data)
data, err = GetInt64("key", func() (int64, error) {
return 0, errors.New("some error")
})
assert.NoError(t, err)
assert.EqualValues(t, 100, data)
Remove("key")
}
+208
View File
@@ -0,0 +1,208 @@
// Copyright 2021 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package cache
import (
"strconv"
"sync"
"time"
"gitea.dev/modules/json"
mc "gitea.com/go-chi/cache" //nolint:depguard // we wrap this package here
lru "github.com/hashicorp/golang-lru/v2"
)
// TwoQueueCache represents a LRU 2Q cache adapter implementation
type TwoQueueCache struct {
lock sync.Mutex
cache *lru.TwoQueueCache[string, any]
interval int
}
// TwoQueueCacheConfig describes the configuration for TwoQueueCache
type TwoQueueCacheConfig struct {
Size int `ini:"SIZE" json:"size"`
RecentRatio float64 `ini:"RECENT_RATIO" json:"recent_ratio"`
GhostRatio float64 `ini:"GHOST_RATIO" json:"ghost_ratio"`
}
// MemoryItem represents a memory cache item.
type MemoryItem struct {
Val any
Created int64
Timeout int64
}
func (item *MemoryItem) hasExpired() bool {
return item.Timeout > 0 &&
(time.Now().Unix()-item.Created) >= item.Timeout
}
var _ mc.Cache = &TwoQueueCache{}
// Put puts value into cache with key and expire time.
func (c *TwoQueueCache) Put(key string, val any, timeout int64) error {
item := &MemoryItem{
Val: val,
Created: time.Now().Unix(),
Timeout: timeout,
}
c.lock.Lock()
defer c.lock.Unlock()
c.cache.Add(key, item)
return nil
}
// Get gets cached value by given key.
func (c *TwoQueueCache) Get(key string) any {
c.lock.Lock()
defer c.lock.Unlock()
cached, ok := c.cache.Get(key)
if !ok {
return nil
}
item, ok := cached.(*MemoryItem)
if !ok || item.hasExpired() {
c.cache.Remove(key)
return nil
}
return item.Val
}
// Delete deletes cached value by given key.
func (c *TwoQueueCache) Delete(key string) error {
c.lock.Lock()
defer c.lock.Unlock()
c.cache.Remove(key)
return nil
}
// Incr increases cached int-type value by given key as a counter.
func (c *TwoQueueCache) Incr(key string) error {
c.lock.Lock()
defer c.lock.Unlock()
cached, ok := c.cache.Get(key)
if !ok {
return nil
}
item, ok := cached.(*MemoryItem)
if !ok || item.hasExpired() {
c.cache.Remove(key)
return nil
}
var err error
item.Val, err = mc.Incr(item.Val)
return err
}
// Decr decreases cached int-type value by given key as a counter.
func (c *TwoQueueCache) Decr(key string) error {
c.lock.Lock()
defer c.lock.Unlock()
cached, ok := c.cache.Get(key)
if !ok {
return nil
}
item, ok := cached.(*MemoryItem)
if !ok || item.hasExpired() {
c.cache.Remove(key)
return nil
}
var err error
item.Val, err = mc.Decr(item.Val)
return err
}
// IsExist returns true if cached value exists.
func (c *TwoQueueCache) IsExist(key string) bool {
c.lock.Lock()
defer c.lock.Unlock()
cached, ok := c.cache.Peek(key)
if !ok {
return false
}
item, ok := cached.(*MemoryItem)
if !ok || item.hasExpired() {
c.cache.Remove(key)
return false
}
return true
}
// Flush deletes all cached data.
func (c *TwoQueueCache) Flush() error {
c.lock.Lock()
defer c.lock.Unlock()
c.cache.Purge()
return nil
}
func (c *TwoQueueCache) checkAndInvalidate(key string) {
c.lock.Lock()
defer c.lock.Unlock()
cached, ok := c.cache.Peek(key)
if !ok {
return
}
item, ok := cached.(*MemoryItem)
if !ok || item.hasExpired() {
c.cache.Remove(key)
}
}
func (c *TwoQueueCache) startGC() {
if c.interval < 0 {
return
}
for _, key := range c.cache.Keys() {
c.checkAndInvalidate(key)
}
time.AfterFunc(time.Duration(c.interval)*time.Second, c.startGC)
}
// StartAndGC starts GC routine based on config string settings.
func (c *TwoQueueCache) StartAndGC(opts mc.Options) error {
var err error
size := 50000
if opts.AdapterConfig != "" {
size, err = strconv.Atoi(opts.AdapterConfig)
}
if err != nil {
if !json.Valid([]byte(opts.AdapterConfig)) {
return err
}
cfg := &TwoQueueCacheConfig{
Size: 50000,
RecentRatio: lru.Default2QRecentRatio,
GhostRatio: lru.Default2QGhostEntries,
}
_ = json.Unmarshal([]byte(opts.AdapterConfig), cfg)
c.cache, err = lru.New2QParams[string, any](cfg.Size, cfg.RecentRatio, cfg.GhostRatio)
} else {
c.cache, err = lru.New2Q[string, any](size)
}
c.interval = opts.Interval
if c.interval > 0 {
go c.startGC()
}
return err
}
// Ping tests if the cache is alive.
func (c *TwoQueueCache) Ping() error {
return mc.GenericPing(c)
}
func init() {
mc.Register("twoqueue", &TwoQueueCache{})
}
+43
View File
@@ -0,0 +1,43 @@
// Copyright 2022 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package cache
import (
"context"
"time"
)
type cacheContextKeyType struct{}
var cacheContextKey = cacheContextKeyType{}
// contextCacheLifetime is the max lifetime of context cache.
// Since context cache is used to cache data in a request level context, 5 minutes is enough.
// If a context cache is used more than 5 minutes, it's probably abused.
const contextCacheLifetime = 5 * time.Minute
func WithCacheContext(ctx context.Context) context.Context {
if c := GetContextCache(ctx); c != nil {
return ctx
}
return context.WithValue(ctx, cacheContextKey, NewEphemeralCache(contextCacheLifetime))
}
func GetContextCache(ctx context.Context) *EphemeralCache {
c, _ := ctx.Value(cacheContextKey).(*EphemeralCache)
return c
}
// GetWithContextCache returns the cache value of the given key in the given context.
// FIXME: in some cases, the "context cache" should not be used, because it has uncontrollable behaviors
// For example, these calls:
// * GetWithContextCache(TargetID) -> OtherCodeCreateModel(TargetID) -> GetWithContextCache(TargetID)
// Will cause the second call is not able to get the correct created target.
// UNLESS it is certain that the target won't be changed during the request, DO NOT use it.
func GetWithContextCache[T, K any](ctx context.Context, groupKey string, targetKey K, f func(context.Context, K) (T, error)) (T, error) {
if c := GetContextCache(ctx); c != nil {
return GetWithEphemeralCache(ctx, c, groupKey, targetKey, f)
}
return f(ctx, targetKey)
}
+50
View File
@@ -0,0 +1,50 @@
// Copyright 2022 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package cache
import (
"context"
"testing"
"time"
"gitea.dev/modules/test"
"github.com/stretchr/testify/assert"
)
func TestWithCacheContext(t *testing.T) {
ctx := WithCacheContext(t.Context())
c := GetContextCache(ctx)
v, _ := c.Get("empty_field", "my_config1")
assert.Nil(t, v)
const field = "system_setting"
v, _ = c.Get(field, "my_config1")
assert.Nil(t, v)
c.Put(field, "my_config1", 1)
v, _ = c.Get(field, "my_config1")
assert.NotNil(t, v)
assert.Equal(t, 1, v.(int))
c.Delete(field, "my_config1")
c.Delete(field, "my_config2") // remove a non-exist key
v, _ = c.Get(field, "my_config1")
assert.Nil(t, v)
vInt, err := GetWithContextCache(ctx, field, "my_config1", func(context.Context, string) (int, error) {
return 1, nil
})
assert.NoError(t, err)
assert.Equal(t, 1, vInt)
v, _ = c.Get(field, "my_config1")
assert.EqualValues(t, 1, v)
defer test.MockVariableValue(&timeNow, func() time.Time {
return time.Now().Add(5 * time.Minute)
})()
v, _ = c.Get(field, "my_config1")
assert.Nil(t, v)
}
+90
View File
@@ -0,0 +1,90 @@
// Copyright 2025 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package cache
import (
"context"
"sync"
"time"
"gitea.dev/modules/log"
"gitea.dev/modules/util"
)
// EphemeralCache is a cache that can be used to store data in a request level context
// This is useful for caching data that is expensive to calculate and is likely to be
// used multiple times in a request.
type EphemeralCache struct {
data map[any]map[any]any
lock sync.RWMutex
created time.Time
checkLifeTime time.Duration
}
var timeNow = time.Now
func NewEphemeralCache(checkLifeTime ...time.Duration) *EphemeralCache {
return &EphemeralCache{
data: make(map[any]map[any]any),
created: timeNow(),
checkLifeTime: util.OptionalArg(checkLifeTime, 0),
}
}
func (cc *EphemeralCache) checkExceededLifeTime(tp, key any) bool {
if cc.checkLifeTime > 0 && timeNow().Sub(cc.created) > cc.checkLifeTime {
log.Warn("EphemeralCache is expired, is highly likely to be abused for long-life tasks: %v, %v", tp, key)
return true
}
return false
}
func (cc *EphemeralCache) Get(tp, key any) (any, bool) {
if cc.checkExceededLifeTime(tp, key) {
return nil, false
}
cc.lock.RLock()
defer cc.lock.RUnlock()
ret, ok := cc.data[tp][key]
return ret, ok
}
func (cc *EphemeralCache) Put(tp, key, value any) {
if cc.checkExceededLifeTime(tp, key) {
return
}
cc.lock.Lock()
defer cc.lock.Unlock()
d := cc.data[tp]
if d == nil {
d = make(map[any]any)
cc.data[tp] = d
}
d[key] = value
}
func (cc *EphemeralCache) Delete(tp, key any) {
if cc.checkExceededLifeTime(tp, key) {
return
}
cc.lock.Lock()
defer cc.lock.Unlock()
delete(cc.data[tp], key)
}
func GetWithEphemeralCache[T, K any](ctx context.Context, c *EphemeralCache, groupKey string, targetKey K, f func(context.Context, K) (T, error)) (T, error) {
v, has := c.Get(groupKey, targetKey)
if vv, ok := v.(T); has && ok {
return vv, nil
}
t, err := f(ctx, targetKey)
if err != nil {
return t, err
}
c.Put(groupKey, targetKey, t)
return t, nil
}
+120
View File
@@ -0,0 +1,120 @@
// Copyright 2024 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package cache
import (
"errors"
"strings"
"gitea.dev/modules/json"
"gitea.dev/modules/setting"
"gitea.dev/modules/util"
chi_cache "gitea.com/go-chi/cache" //nolint:depguard // we wrap this package here
)
type GetJSONError struct {
err error
cachedError string // Golang error can't be stored in cache, only the string message could be stored
}
func (e *GetJSONError) ToError() error {
if e.err != nil {
return e.err
}
return errors.New("cached error: " + e.cachedError)
}
type StringCache interface {
Ping() error
Get(key string) (string, bool)
Put(key, value string, ttl int64) error
Delete(key string) error
IsExist(key string) bool
PutJSON(key string, v any, ttl int64) error
GetJSON(key string, ptr any) (exist bool, err *GetJSONError)
ChiCache() chi_cache.Cache
}
type stringCache struct {
chiCache chi_cache.Cache
}
func NewStringCache(cacheConfig setting.Cache) (StringCache, error) {
adapter := util.IfZero(cacheConfig.Adapter, "memory")
interval := util.IfZero(cacheConfig.Interval, 60)
cc, err := chi_cache.NewCacher(chi_cache.Options{
Adapter: adapter,
AdapterConfig: cacheConfig.Conn,
Interval: interval,
})
if err != nil {
return nil, err
}
return &stringCache{chiCache: cc}, nil
}
func (sc *stringCache) Ping() error {
return sc.chiCache.Ping()
}
func (sc *stringCache) Get(key string) (string, bool) {
v := sc.chiCache.Get(key)
if v == nil {
return "", false
}
s, ok := v.(string)
return s, ok
}
func (sc *stringCache) Put(key, value string, ttl int64) error {
return sc.chiCache.Put(key, value, ttl)
}
func (sc *stringCache) Delete(key string) error {
return sc.chiCache.Delete(key)
}
func (sc *stringCache) IsExist(key string) bool {
return sc.chiCache.IsExist(key)
}
const cachedErrorPrefix = "<CACHED-ERROR>:"
func (sc *stringCache) PutJSON(key string, v any, ttl int64) error {
var s string
switch v := v.(type) {
case error:
s = cachedErrorPrefix + v.Error()
default:
b, err := json.Marshal(v)
if err != nil {
return err
}
s = util.UnsafeBytesToString(b)
}
return sc.chiCache.Put(key, s, ttl)
}
func (sc *stringCache) GetJSON(key string, ptr any) (exist bool, getErr *GetJSONError) {
s, ok := sc.Get(key)
if !ok || s == "" {
return false, nil
}
s, isCachedError := strings.CutPrefix(s, cachedErrorPrefix)
if isCachedError {
return true, &GetJSONError{cachedError: s}
}
if err := json.Unmarshal(util.UnsafeStringToBytes(s), ptr); err != nil {
return false, &GetJSONError{err: err}
}
return true, nil
}
func (sc *stringCache) ChiCache() chi_cache.Cache {
return sc.chiCache
}