初始提交: Gitea 项目代码
This commit is contained in:
root
@@ -0,0 +1,85 @@
|
||||
// Copyright 2020 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package stats
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
repo_model "gitea.dev/models/repo"
|
||||
"gitea.dev/modules/git"
|
||||
"gitea.dev/modules/git/languagestats"
|
||||
"gitea.dev/modules/gitrepo"
|
||||
"gitea.dev/modules/graceful"
|
||||
"gitea.dev/modules/log"
|
||||
"gitea.dev/modules/process"
|
||||
"gitea.dev/modules/setting"
|
||||
)
|
||||
|
||||
// DBIndexer implements Indexer interface to use database's like search
|
||||
type DBIndexer struct{}
|
||||
|
||||
// Index repository status function
|
||||
func (db *DBIndexer) Index(id int64) error {
|
||||
ctx, _, finished := process.GetManager().AddContext(graceful.GetManager().ShutdownContext(), fmt.Sprintf("Stats.DB Index Repo[%d]", id))
|
||||
defer finished()
|
||||
|
||||
repo, err := repo_model.GetRepositoryByID(ctx, id)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if repo.IsEmpty {
|
||||
return nil
|
||||
}
|
||||
|
||||
status, err := repo_model.GetIndexerStatus(ctx, repo, repo_model.RepoIndexerTypeStats)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
gitRepo, err := gitrepo.OpenRepository(ctx, repo)
|
||||
if err != nil {
|
||||
if err.Error() == "no such file or directory" {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
defer gitRepo.Close()
|
||||
|
||||
// Get latest commit for default branch
|
||||
commitID, err := gitRepo.GetBranchCommitID(repo.DefaultBranch)
|
||||
if err != nil {
|
||||
if git.IsErrBranchNotExist(err) || git.IsErrNotExist(err) || setting.IsInTesting {
|
||||
log.Debug("Unable to get commit ID for default branch %s in %s ... skipping this repository", repo.DefaultBranch, repo.FullName())
|
||||
return nil
|
||||
}
|
||||
log.Error("Unable to get commit ID for default branch %s in %s. Error: %v", repo.DefaultBranch, repo.FullName(), err)
|
||||
return err
|
||||
}
|
||||
|
||||
// Do not recalculate stats if already calculated for this commit
|
||||
if status.CommitSha == commitID {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Calculate and save language statistics to database
|
||||
stats, err := languagestats.GetLanguageStats(gitRepo, commitID)
|
||||
if err != nil {
|
||||
if !setting.IsInTesting {
|
||||
log.Error("Unable to get language stats for ID %s for default branch %s in %s. Error: %v", commitID, repo.DefaultBranch, repo.FullName(), err)
|
||||
}
|
||||
return err
|
||||
}
|
||||
err = repo_model.UpdateLanguageStats(ctx, repo, commitID, stats)
|
||||
if err != nil {
|
||||
log.Error("Unable to update language stats for ID %s for default branch %s in %s. Error: %v", commitID, repo.DefaultBranch, repo.FullName(), err)
|
||||
return err
|
||||
}
|
||||
|
||||
log.Debug("DBIndexer completed language stats for ID %s for default branch %s in %s. stats count: %d", commitID, repo.DefaultBranch, repo.FullName(), len(stats))
|
||||
return nil
|
||||
}
|
||||
|
||||
// Close dummy function
|
||||
func (db *DBIndexer) Close() {
|
||||
}
|
||||
@@ -0,0 +1,88 @@
|
||||
// Copyright 2020 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package stats
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"gitea.dev/models/db"
|
||||
repo_model "gitea.dev/models/repo"
|
||||
"gitea.dev/modules/graceful"
|
||||
"gitea.dev/modules/log"
|
||||
)
|
||||
|
||||
// Indexer defines an interface to index repository stats
|
||||
// TODO: this indexer is quite different from the others, maybe this package should be moved out from module/indexer
|
||||
type Indexer interface {
|
||||
Index(id int64) error
|
||||
Close()
|
||||
}
|
||||
|
||||
// indexer represents a indexer instance
|
||||
var indexer Indexer
|
||||
|
||||
// Init initialize the repo indexer
|
||||
func Init() error {
|
||||
indexer = &DBIndexer{}
|
||||
|
||||
if err := initStatsQueue(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
go populateRepoIndexer(graceful.GetManager().ShutdownContext())
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// populateRepoIndexer populate the repo indexer with pre-existing data. This
|
||||
// should only be run when the indexer is created for the first time.
|
||||
func populateRepoIndexer(ctx context.Context) {
|
||||
log.Info("Populating the repo stats indexer with existing repositories")
|
||||
|
||||
isShutdown := graceful.GetManager().IsShutdown()
|
||||
|
||||
exist, err := db.IsTableNotEmpty("repository")
|
||||
if err != nil {
|
||||
log.Fatal("System error: %v", err)
|
||||
} else if !exist {
|
||||
return
|
||||
}
|
||||
|
||||
var maxRepoID int64
|
||||
if maxRepoID, err = db.GetMaxID("repository"); err != nil {
|
||||
log.Fatal("System error: %v", err)
|
||||
}
|
||||
|
||||
// start with the maximum existing repo ID and work backwards, so that we
|
||||
// don't include repos that are created after gitea starts; such repos will
|
||||
// already be added to the indexer, and we don't need to add them again.
|
||||
for maxRepoID > 0 {
|
||||
select {
|
||||
case <-isShutdown:
|
||||
log.Info("Repository Stats Indexer population shutdown before completion")
|
||||
return
|
||||
default:
|
||||
}
|
||||
ids, err := repo_model.GetUnindexedRepos(ctx, repo_model.RepoIndexerTypeStats, maxRepoID, 0, 50)
|
||||
if err != nil {
|
||||
log.Error("populateRepoIndexer: %v", err)
|
||||
return
|
||||
} else if len(ids) == 0 {
|
||||
break
|
||||
}
|
||||
for _, id := range ids {
|
||||
select {
|
||||
case <-isShutdown:
|
||||
log.Info("Repository Stats Indexer population shutdown before completion")
|
||||
return
|
||||
default:
|
||||
}
|
||||
if err := statsQueue.Push(id); err != nil {
|
||||
log.Error("statsQueue.Push: %v", err)
|
||||
}
|
||||
maxRepoID = id - 1
|
||||
}
|
||||
}
|
||||
log.Info("Done (re)populating the repo stats indexer with existing repositories")
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
// Copyright 2020 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package stats
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
repo_model "gitea.dev/models/repo"
|
||||
"gitea.dev/models/unittest"
|
||||
"gitea.dev/modules/queue"
|
||||
"gitea.dev/modules/setting"
|
||||
|
||||
_ "gitea.dev/models"
|
||||
_ "gitea.dev/models/actions"
|
||||
_ "gitea.dev/models/activities"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestMain(m *testing.M) {
|
||||
unittest.MainTest(m)
|
||||
}
|
||||
|
||||
func TestRepoStatsIndex(t *testing.T) {
|
||||
assert.NoError(t, unittest.PrepareTestDatabase())
|
||||
setting.CfgProvider, _ = setting.NewConfigProviderFromData("")
|
||||
|
||||
setting.LoadQueueSettings()
|
||||
|
||||
err := Init()
|
||||
assert.NoError(t, err)
|
||||
|
||||
repo, err := repo_model.GetRepositoryByID(t.Context(), 1)
|
||||
assert.NoError(t, err)
|
||||
|
||||
err = UpdateRepoIndexer(repo)
|
||||
assert.NoError(t, err)
|
||||
|
||||
assert.NoError(t, queue.GetManager().FlushAll(t.Context(), 5*time.Second))
|
||||
|
||||
status, err := repo_model.GetIndexerStatus(t.Context(), repo, repo_model.RepoIndexerTypeStats)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, "65f1bf27bc3bf70f64657658635e66094edbcb4d", status.CommitSha)
|
||||
langs, err := repo_model.GetTopLanguageStats(t.Context(), repo, 5)
|
||||
assert.NoError(t, err)
|
||||
assert.Empty(t, langs)
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
// Copyright 2020 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package stats
|
||||
|
||||
import (
|
||||
"errors"
|
||||
|
||||
repo_model "gitea.dev/models/repo"
|
||||
"gitea.dev/modules/graceful"
|
||||
"gitea.dev/modules/log"
|
||||
"gitea.dev/modules/queue"
|
||||
"gitea.dev/modules/setting"
|
||||
)
|
||||
|
||||
// statsQueue represents a queue to handle repository stats updates
|
||||
var statsQueue *queue.WorkerPoolQueue[int64]
|
||||
|
||||
// handle passed PR IDs and test the PRs
|
||||
func handler(items ...int64) []int64 {
|
||||
for _, opts := range items {
|
||||
if err := indexer.Index(opts); err != nil {
|
||||
if !setting.IsInTesting {
|
||||
log.Error("stats queue indexer.Index(%d) failed: %v", opts, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func initStatsQueue() error {
|
||||
statsQueue = queue.CreateUniqueQueue(graceful.GetManager().ShutdownContext(), "repo_stats_update", handler)
|
||||
if statsQueue == nil {
|
||||
return errors.New("unable to create repo_stats_update queue")
|
||||
}
|
||||
go graceful.GetManager().RunWithCancel(statsQueue)
|
||||
return nil
|
||||
}
|
||||
|
||||
// UpdateRepoIndexer update a repository's entries in the indexer
|
||||
func UpdateRepoIndexer(repo *repo_model.Repository) error {
|
||||
if err := statsQueue.Push(repo.ID); err != nil {
|
||||
if err != queue.ErrAlreadyInQueue {
|
||||
return err
|
||||
}
|
||||
log.Debug("Repo ID: %d already queued", repo.ID)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
Reference in New Issue
Block a user