初始提交: Gitea 项目代码

services/doctor/dbconsistency.go

@@ -0,0 +1,274 @@
+// Copyright 2020 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package doctor
+
+import (
+	"context"
+
+	actions_model "gitea.dev/models/actions"
+	activities_model "gitea.dev/models/activities"
+	"gitea.dev/models/db"
+	issues_model "gitea.dev/models/issues"
+	"gitea.dev/models/migrations"
+	repo_model "gitea.dev/models/repo"
+	secret_model "gitea.dev/models/secret"
+	"gitea.dev/modules/log"
+	"gitea.dev/modules/setting"
+	issue_service "gitea.dev/services/issue"
+)
+
+type consistencyCheck struct {
+	Name         string
+	Counter      func(context.Context) (int64, error)
+	Fixer        func(context.Context) (int64, error)
+	FixedMessage string
+}
+
+func (c *consistencyCheck) Run(ctx context.Context, logger log.Logger, autofix bool) error {
+	count, err := c.Counter(ctx)
+	if err != nil {
+		logger.Critical("Error: %v whilst counting %s", err, c.Name)
+		return err
+	}
+	if count > 0 {
+		if autofix {
+			var fixed int64
+			if fixed, err = c.Fixer(ctx); err != nil {
+				logger.Critical("Error: %v whilst fixing %s", err, c.Name)
+				return err
+			}
+
+			prompt := "Deleted"
+			if c.FixedMessage != "" {
+				prompt = c.FixedMessage
+			}
+
+			if fixed < 0 {
+				logger.Info(prompt+" %d %s", count, c.Name)
+			} else {
+				logger.Info(prompt+" %d/%d %s", fixed, count, c.Name)
+			}
+		} else {
+			logger.Warn("Found %d %s", count, c.Name)
+		}
+	}
+	return nil
+}
+
+func asFixer(fn func(ctx context.Context) error) func(ctx context.Context) (int64, error) {
+	return func(ctx context.Context) (int64, error) {
+		err := fn(ctx)
+		return -1, err
+	}
+}
+
+func genericOrphanCheck(name, subject, refObject, joinCond string) consistencyCheck {
+	return consistencyCheck{
+		Name: name,
+		Counter: func(ctx context.Context) (int64, error) {
+			return db.CountOrphanedObjects(ctx, subject, refObject, joinCond)
+		},
+		Fixer: func(ctx context.Context) (int64, error) {
+			err := db.DeleteOrphanedObjects(ctx, subject, refObject, joinCond)
+			return -1, err
+		},
+	}
+}
+
+func prepareDBConsistencyChecks() []consistencyCheck {
+	consistencyChecks := []consistencyCheck{
+		{
+			// find labels without existing repo or org
+			Name:    "Orphaned Labels without existing repository or organisation",
+			Counter: issues_model.CountOrphanedLabels,
+			Fixer:   asFixer(issues_model.DeleteOrphanedLabels),
+		},
+		{
+			// find IssueLabels without existing label
+			Name:    "Orphaned Issue Labels without existing label",
+			Counter: issues_model.CountOrphanedIssueLabels,
+			Fixer:   asFixer(issues_model.DeleteOrphanedIssueLabels),
+		},
+		{
+			// find issues without existing repository
+			Name:    "Orphaned Issues without existing repository",
+			Counter: issues_model.CountOrphanedIssues,
+			Fixer:   asFixer(issue_service.DeleteOrphanedIssues),
+		},
+		// find releases without existing repository
+		genericOrphanCheck("Orphaned Releases without existing repository",
+			"release", "repository", "`release`.repo_id=repository.id"),
+		// find pulls without existing issues
+		genericOrphanCheck("Orphaned PullRequests without existing issue",
+			"pull_request", "issue", "pull_request.issue_id=issue.id"),
+		// find pull requests without base repository
+		genericOrphanCheck("Pull request entries without existing base repository",
+			"pull_request", "repository", "pull_request.base_repo_id=repository.id"),
+		// find tracked times without existing issues/pulls
+		genericOrphanCheck("Orphaned TrackedTimes without existing issue",
+			"tracked_time", "issue", "tracked_time.issue_id=issue.id"),
+		// find attachments without existing issues or releases
+		{
+			Name:    "Orphaned Attachments without existing issues or releases",
+			Counter: repo_model.CountOrphanedAttachments,
+			Fixer:   asFixer(repo_model.DeleteOrphanedAttachments),
+		},
+		// find null archived repositories
+		{
+			Name:         "Repositories with is_archived IS NULL",
+			Counter:      repo_model.CountNullArchivedRepository,
+			Fixer:        repo_model.FixNullArchivedRepository,
+			FixedMessage: "Fixed",
+		},
+		// find label comments with empty labels
+		{
+			Name:         "Label comments with empty labels",
+			Counter:      issues_model.CountCommentTypeLabelWithEmptyLabel,
+			Fixer:        issues_model.FixCommentTypeLabelWithEmptyLabel,
+			FixedMessage: "Fixed",
+		},
+		// find label comments with labels from outside the repository
+		{
+			Name:         "Label comments with labels from outside the repository",
+			Counter:      issues_model.CountCommentTypeLabelWithOutsideLabels,
+			Fixer:        issues_model.FixCommentTypeLabelWithOutsideLabels,
+			FixedMessage: "Removed",
+		},
+		// find issue_label with labels from outside the repository
+		{
+			Name:         "IssueLabels with Labels from outside the repository",
+			Counter:      issues_model.CountIssueLabelWithOutsideLabels,
+			Fixer:        issues_model.FixIssueLabelWithOutsideLabels,
+			FixedMessage: "Removed",
+		},
+		{
+			Name:         "Action with created_unix set as an empty string",
+			Counter:      activities_model.CountActionCreatedUnixString,
+			Fixer:        activities_model.FixActionCreatedUnixString,
+			FixedMessage: "Set to zero",
+		},
+		{
+			Name:         "Action Runners without existing owner",
+			Counter:      actions_model.CountRunnersWithoutBelongingOwner,
+			Fixer:        actions_model.FixRunnersWithoutBelongingOwner,
+			FixedMessage: "Removed",
+		},
+		{
+			Name:         "Action Runners without existing repository",
+			Counter:      actions_model.CountRunnersWithoutBelongingRepo,
+			Fixer:        actions_model.FixRunnersWithoutBelongingRepo,
+			FixedMessage: "Removed",
+		},
+		{
+			Name:         "Topics with empty repository count",
+			Counter:      repo_model.CountOrphanedTopics,
+			Fixer:        repo_model.DeleteOrphanedTopics,
+			FixedMessage: "Removed",
+		},
+		{
+			Name:         "Repository level Runners with non-zero owner_id",
+			Counter:      actions_model.CountWrongRepoLevelRunners,
+			Fixer:        actions_model.UpdateWrongRepoLevelRunners,
+			FixedMessage: "Corrected",
+		},
+		{
+			Name:         "Repository level Variables with non-zero owner_id",
+			Counter:      actions_model.CountWrongRepoLevelVariables,
+			Fixer:        actions_model.UpdateWrongRepoLevelVariables,
+			FixedMessage: "Corrected",
+		},
+		{
+			Name:         "Repository level Secrets with non-zero owner_id",
+			Counter:      secret_model.CountWrongRepoLevelSecrets,
+			Fixer:        secret_model.UpdateWrongRepoLevelSecrets,
+			FixedMessage: "Corrected",
+		},
+	}
+
+	// TODO: function to recalc all counters
+
+	if setting.Database.Type.IsPostgreSQL() {
+		consistencyChecks = append(consistencyChecks, consistencyCheck{
+			Name:         "Sequence values",
+			Counter:      db.CountBadSequences,
+			Fixer:        asFixer(db.FixBadSequences),
+			FixedMessage: "Updated",
+		})
+	}
+
+	consistencyChecks = append(consistencyChecks,
+		// find protected branches without existing repository
+		genericOrphanCheck("Protected Branches without existing repository",
+			"protected_branch", "repository", "protected_branch.repo_id=repository.id"),
+		// find branches without existing repository
+		genericOrphanCheck("Branches without existing repository",
+			"branch", "repository", "branch.repo_id=repository.id"),
+		// find LFS locks without existing repository
+		genericOrphanCheck("LFS locks without existing repository",
+			"lfs_lock", "repository", "lfs_lock.repo_id=repository.id"),
+		// find collaborations without users
+		genericOrphanCheck("Collaborations without existing user",
+			"collaboration", "user", "collaboration.user_id=`user`.id"),
+		// find collaborations without repository
+		genericOrphanCheck("Collaborations without existing repository",
+			"collaboration", "repository", "collaboration.repo_id=repository.id"),
+		// find access without users
+		genericOrphanCheck("Access entries without existing user",
+			"access", "user", "access.user_id=`user`.id"),
+		// find access without repository
+		genericOrphanCheck("Access entries without existing repository",
+			"access", "repository", "access.repo_id=repository.id"),
+		// find action without repository
+		genericOrphanCheck("Action entries without existing repository",
+			"action", "repository", "action.repo_id=repository.id"),
+		// find action without user
+		genericOrphanCheck("Action entries without existing user",
+			"action", "user", "action.act_user_id=`user`.id"),
+		// find OAuth2Grant without existing user
+		genericOrphanCheck("Orphaned OAuth2Grant without existing User",
+			"oauth2_grant", "user", "oauth2_grant.user_id=`user`.id"),
+		// find OAuth2Application without existing user
+		genericOrphanCheck("Orphaned OAuth2Application without existing User",
+			"oauth2_application", "user", "oauth2_application.uid=0 OR oauth2_application.uid=`user`.id"),
+		// find OAuth2AuthorizationCode without existing OAuth2Grant
+		genericOrphanCheck("Orphaned OAuth2AuthorizationCode without existing OAuth2Grant",
+			"oauth2_authorization_code", "oauth2_grant", "oauth2_authorization_code.grant_id=oauth2_grant.id"),
+		// find stopwatches without existing user
+		genericOrphanCheck("Orphaned Stopwatches without existing User",
+			"stopwatch", "user", "stopwatch.user_id=`user`.id"),
+		// find stopwatches without existing issue
+		genericOrphanCheck("Orphaned Stopwatches without existing Issue",
+			"stopwatch", "issue", "stopwatch.issue_id=`issue`.id"),
+		// find redirects without existing user.
+		genericOrphanCheck("Orphaned Redirects without existing redirect user",
+			"user_redirect", "user", "user_redirect.redirect_user_id=`user`.id"),
+	)
+	return consistencyChecks
+}
+
+func checkDBConsistency(ctx context.Context, logger log.Logger, autofix bool) error {
+	// make sure DB version is uptodate
+	if err := db.InitEngineWithMigration(ctx, migrations.EnsureUpToDate); err != nil {
+		logger.Critical("Model version on the database does not match the current Gitea version. Model consistency will not be checked until the database is upgraded")
+		return err
+	}
+	consistencyChecks := prepareDBConsistencyChecks()
+	for _, c := range consistencyChecks {
+		if err := c.Run(ctx, logger, autofix); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func init() {
+	Register(&Check{
+		Title:     "Check consistency of database",
+		Name:      "check-db-consistency",
+		IsDefault: false,
+		Run:       checkDBConsistency,
+		Priority:  3,
+	})
+}

services/doctor/dbconsistency_test.go

@@ -0,0 +1,50 @@
+// Copyright 2024 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package doctor
+
+import (
+	"slices"
+	"testing"
+
+	"gitea.dev/models/auth"
+	"gitea.dev/models/db"
+	"gitea.dev/models/unittest"
+	"gitea.dev/models/user"
+	"gitea.dev/modules/log"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestConsistencyCheck(t *testing.T) {
+	checks := prepareDBConsistencyChecks()
+	idx := slices.IndexFunc(checks, func(check consistencyCheck) bool {
+		return check.Name == "Orphaned OAuth2Application without existing User"
+	})
+	require.NotEqual(t, -1, idx)
+
+	_ = db.TruncateBeans(t.Context(), &auth.OAuth2Application{}, &user.User{})
+	_ = db.TruncateBeans(t.Context(), &auth.OAuth2Application{}, &auth.OAuth2Application{})
+
+	err := db.Insert(t.Context(), &user.User{ID: 1})
+	assert.NoError(t, err)
+	err = db.Insert(t.Context(), &auth.OAuth2Application{Name: "test-oauth2-app-1", ClientID: "client-id-1"})
+	assert.NoError(t, err)
+	err = db.Insert(t.Context(), &auth.OAuth2Application{Name: "test-oauth2-app-2", ClientID: "client-id-2", UID: 1})
+	assert.NoError(t, err)
+	err = db.Insert(t.Context(), &auth.OAuth2Application{Name: "test-oauth2-app-3", ClientID: "client-id-3", UID: 99999999})
+	assert.NoError(t, err)
+
+	unittest.AssertExistsAndLoadBean(t, &auth.OAuth2Application{ClientID: "client-id-1"})
+	unittest.AssertExistsAndLoadBean(t, &auth.OAuth2Application{ClientID: "client-id-2"})
+	unittest.AssertExistsAndLoadBean(t, &auth.OAuth2Application{ClientID: "client-id-3"})
+
+	oauth2AppCheck := checks[idx]
+	err = oauth2AppCheck.Run(t.Context(), log.GetManager().GetLogger(log.DEFAULT), true)
+	assert.NoError(t, err)
+
+	unittest.AssertExistsAndLoadBean(t, &auth.OAuth2Application{ClientID: "client-id-1"})
+	unittest.AssertExistsAndLoadBean(t, &auth.OAuth2Application{ClientID: "client-id-2"})
+	unittest.AssertNotExistsBean(t, &auth.OAuth2Application{ClientID: "client-id-3"})
+}

services/doctor/doctor.go

@@ -0,0 +1,138 @@
+// Copyright 2020 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package doctor
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"sort"
+	"strings"
+
+	"gitea.dev/models/db"
+	"gitea.dev/modules/git"
+	"gitea.dev/modules/log"
+	"gitea.dev/modules/setting"
+	"gitea.dev/modules/storage"
+)
+
+// Check represents a Doctor check
+type Check struct {
+	Title                      string
+	Name                       string
+	IsDefault                  bool
+	Run                        func(ctx context.Context, logger log.Logger, autofix bool) error
+	AbortIfFailed              bool
+	SkipDatabaseInitialization bool
+	Priority                   int
+	InitStorage                bool
+}
+
+func initDBSkipLogger(ctx context.Context) error {
+	setting.MustInstalled()
+	setting.LoadDBSetting()
+	if err := db.InitEngine(ctx); err != nil {
+		return fmt.Errorf("db.InitEngine: %w", err)
+	}
+	// some doctor sub-commands need to use git command
+	if err := git.InitFull(); err != nil {
+		return fmt.Errorf("git.InitFull: %w", err)
+	}
+	return nil
+}
+
+type doctorCheckLogger struct {
+	colorize bool
+}
+
+var _ log.BaseLogger = (*doctorCheckLogger)(nil)
+
+func (d *doctorCheckLogger) Log(skip int, event *log.Event, format string, v ...any) {
+	_, _ = fmt.Fprintf(os.Stdout, format+"\n", v...)
+}
+
+func (d *doctorCheckLogger) GetLevel() log.Level {
+	return log.TRACE
+}
+
+type doctorCheckStepLogger struct {
+	colorize bool
+}
+
+var _ log.BaseLogger = (*doctorCheckStepLogger)(nil)
+
+func (d *doctorCheckStepLogger) Log(skip int, event *log.Event, format string, v ...any) {
+	levelChar := fmt.Sprintf("[%s]", strings.ToUpper(event.Level.String()[0:1]))
+	var levelArg any = levelChar
+	if d.colorize {
+		levelArg = log.NewColoredValue(levelChar, event.Level.ColorAttributes()...)
+	}
+	args := append([]any{levelArg}, v...)
+	_, _ = fmt.Fprintf(os.Stdout, " - %s "+format+"\n", args...)
+}
+
+func (d *doctorCheckStepLogger) GetLevel() log.Level {
+	return log.TRACE
+}
+
+// Checks is the list of available commands
+var Checks []*Check
+
+// RunChecks runs the doctor checks for the provided list
+func RunChecks(ctx context.Context, colorize, autofix bool, checks []*Check) error {
+	SortChecks(checks)
+	// the checks output logs by a special logger, they do not use the default logger
+	logger := log.BaseLoggerToGeneralLogger(&doctorCheckLogger{colorize: colorize})
+	loggerStep := log.BaseLoggerToGeneralLogger(&doctorCheckStepLogger{colorize: colorize})
+	dbIsInit := false
+	storageIsInit := false
+	for i, check := range checks {
+		if !dbIsInit && !check.SkipDatabaseInitialization {
+			// Only open database after the most basic configuration check
+			if err := initDBSkipLogger(ctx); err != nil {
+				logger.Error("Error whilst initializing the database: %v", err)
+				logger.Error("Check if you are using the right config file. You can use a --config directive to specify one.")
+				return nil
+			}
+			dbIsInit = true
+		}
+		if !storageIsInit && check.InitStorage {
+			if err := storage.Init(); err != nil {
+				logger.Error("Error whilst initializing the storage: %v", err)
+				logger.Error("Check if you are using the right config file. You can use a --config directive to specify one.")
+				return nil
+			}
+			storageIsInit = true
+		}
+		logger.Info("\n[%d] %s", i+1, check.Title)
+		if err := check.Run(ctx, loggerStep, autofix); err != nil {
+			if check.AbortIfFailed {
+				logger.Critical("FAIL")
+				return err
+			}
+			logger.Error("ERROR")
+		} else {
+			logger.Info("OK")
+		}
+	}
+	logger.Info("\nAll done (checks: %d).", len(checks))
+	return nil
+}
+
+// Register registers a command with the list
+func Register(command *Check) {
+	Checks = append(Checks, command)
+}
+
+func SortChecks(checks []*Check) {
+	sort.SliceStable(checks, func(i, j int) bool {
+		if checks[i].Priority == checks[j].Priority {
+			return checks[i].Name < checks[j].Name
+		}
+		if checks[i].Priority == 0 {
+			return false
+		}
+		return checks[i].Priority < checks[j].Priority
+	})
+}

services/doctor/lfs.go

@@ -0,0 +1,51 @@
+// Copyright 2022 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package doctor
+
+import (
+	"context"
+	"errors"
+	"time"
+
+	"gitea.dev/modules/log"
+	"gitea.dev/modules/setting"
+	"gitea.dev/services/repository"
+)
+
+func init() {
+	Register(&Check{
+		Title:                      "Garbage collect LFS",
+		Name:                       "gc-lfs",
+		IsDefault:                  false,
+		Run:                        garbageCollectLFSCheck,
+		AbortIfFailed:              false,
+		SkipDatabaseInitialization: false,
+		Priority:                   1,
+	})
+}
+
+func garbageCollectLFSCheck(ctx context.Context, logger log.Logger, autofix bool) error {
+	if !setting.LFS.StartServer {
+		return errors.New("LFS support is disabled")
+	}
+
+	if err := repository.GarbageCollectLFSMetaObjects(ctx, repository.GarbageCollectLFSMetaObjectsOptions{
+		LogDetail: logger.Info,
+		AutoFix:   autofix,
+		// Only attempt to garbage collect lfs meta objects older than a week as the order of git lfs upload
+		// and git object upload is not necessarily guaranteed. It's possible to imagine a situation whereby
+		// an LFS object is uploaded but the git branch is not uploaded immediately, or there are some rapid
+		// changes in new branches that might lead to lfs objects becoming temporarily unassociated with git
+		// objects.
+		//
+		// It is likely that a week is potentially excessive but it should definitely be enough that any
+		// unassociated LFS object is genuinely unassociated.
+		OlderThan: time.Now().Add(-24 * time.Hour * 7),
+		// We don't set the UpdatedLessRecentlyThan because we want to do a full GC
+	}); err != nil {
+		return err
+	}
+
+	return checkStorage(&checkStorageOptions{LFS: true})(ctx, logger, autofix)
+}

services/doctor/main_test.go

@@ -0,0 +1,14 @@
+// Copyright 2024 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package doctor
+
+import (
+	"testing"
+
+	"gitea.dev/models/unittest"
+)
+
+func TestMain(m *testing.M) {
+	unittest.MainTest(m)
+}

services/doctor/misc.go

@@ -0,0 +1,225 @@
+// Copyright 2020 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package doctor
+
+import (
+	"context"
+	"fmt"
+
+	"gitea.dev/models"
+	"gitea.dev/models/db"
+	repo_model "gitea.dev/models/repo"
+	user_model "gitea.dev/models/user"
+	"gitea.dev/modules/gitrepo"
+	"gitea.dev/modules/log"
+	"gitea.dev/modules/structs"
+
+	lru "github.com/hashicorp/golang-lru/v2"
+	"xorm.io/builder"
+)
+
+func iterateRepositories(ctx context.Context, each func(*repo_model.Repository) error) error {
+	err := db.Iterate(
+		ctx,
+		builder.Gt{"id": 0},
+		func(ctx context.Context, bean *repo_model.Repository) error {
+			return each(bean)
+		},
+	)
+	return err
+}
+
+func checkHooks(ctx context.Context, logger log.Logger, autofix bool) error {
+	if err := iterateRepositories(ctx, func(repo *repo_model.Repository) error {
+		results, err := gitrepo.CheckDelegateHooks(ctx, repo)
+		if err != nil {
+			logger.Critical("Unable to check delegate hooks for repo %-v. ERROR: %v", repo, err)
+			return fmt.Errorf("Unable to check delegate hooks for repo %-v. ERROR: %w", repo, err)
+		}
+		if len(results) > 0 && autofix {
+			logger.Warn("Regenerated hooks for %s", repo.FullName())
+			if err := gitrepo.CreateDelegateHooks(ctx, repo); err != nil {
+				logger.Critical("Unable to recreate delegate hooks for %-v. ERROR: %v", repo, err)
+				return fmt.Errorf("Unable to recreate delegate hooks for %-v. ERROR: %w", repo, err)
+			}
+		}
+		for _, result := range results {
+			logger.Warn(result)
+		}
+		return nil
+	}); err != nil {
+		logger.Critical("Errors noted whilst checking delegate hooks.")
+		return err
+	}
+	return nil
+}
+
+func checkUserStarNum(ctx context.Context, logger log.Logger, autofix bool) error {
+	if autofix {
+		if err := models.DoctorUserStarNum(ctx); err != nil {
+			logger.Critical("Unable update User Stars numbers")
+			return err
+		}
+		logger.Info("Updated User Stars numbers.")
+	} else {
+		logger.Info("No check available for User Stars numbers (skipped)")
+	}
+	return nil
+}
+
+func checkDaemonExport(ctx context.Context, logger log.Logger, autofix bool) error {
+	numRepos := 0
+	numNeedUpdate := 0
+	cache, err := lru.New[int64, any](512)
+	if err != nil {
+		logger.Critical("Unable to create cache: %v", err)
+		return err
+	}
+	if err := iterateRepositories(ctx, func(repo *repo_model.Repository) error {
+		numRepos++
+
+		if owner, has := cache.Get(repo.OwnerID); has {
+			repo.Owner = owner.(*user_model.User)
+		} else {
+			if err := repo.LoadOwner(ctx); err != nil {
+				return err
+			}
+			cache.Add(repo.OwnerID, repo.Owner)
+		}
+
+		// Create/Remove git-daemon-export-ok for git-daemon...
+		daemonExportFile := `git-daemon-export-ok`
+		isExist, err := gitrepo.IsRepoFileExist(ctx, repo, daemonExportFile)
+		if err != nil {
+			log.Error("Unable to check if %s:%s exists. Error: %v", repo.FullName(), daemonExportFile, err)
+			return err
+		}
+		isPublic := !repo.IsPrivate && repo.Owner.Visibility == structs.VisibleTypePublic
+
+		if isPublic != isExist {
+			numNeedUpdate++
+			if autofix {
+				if !isPublic && isExist {
+					if err = gitrepo.RemoveRepoFileOrDir(ctx, repo, daemonExportFile); err != nil {
+						log.Error("Failed to remove %s:%s: %v", repo.FullName(), daemonExportFile, err)
+					}
+				} else if isPublic && !isExist {
+					if f, err := gitrepo.CreateRepoFile(ctx, repo, daemonExportFile); err != nil {
+						log.Error("Failed to create %s:%s: %v", repo.FullName(), daemonExportFile, err)
+					} else {
+						f.Close()
+					}
+				}
+			}
+		}
+		return nil
+	}); err != nil {
+		logger.Critical("Unable to checkDaemonExport: %v", err)
+		return err
+	}
+
+	if autofix {
+		logger.Info("Updated git-daemon-export-ok files for %d of %d repositories.", numNeedUpdate, numRepos)
+	} else {
+		logger.Info("Checked %d repositories, %d need updates.", numRepos, numNeedUpdate)
+	}
+
+	return nil
+}
+
+func checkCommitGraph(ctx context.Context, logger log.Logger, autofix bool) error {
+	numRepos := 0
+	numNeedUpdate := 0
+	numWritten := 0
+	if err := iterateRepositories(ctx, func(repo *repo_model.Repository) error {
+		numRepos++
+
+		commitGraphExists := func() (bool, error) {
+			// Check commit-graph exists
+			commitGraphFile := `objects/info/commit-graph`
+			isExist, err := gitrepo.IsRepoFileExist(ctx, repo, commitGraphFile)
+			if err != nil {
+				logger.Error("Unable to check if %s exists. Error: %v", commitGraphFile, err)
+				return false, err
+			}
+
+			if !isExist {
+				commitGraphsDir := `objects/info/commit-graphs`
+				isExist, err = gitrepo.IsRepoDirExist(ctx, repo, commitGraphsDir)
+				if err != nil {
+					logger.Error("Unable to check if %s exists. Error: %v", commitGraphsDir, err)
+					return false, err
+				}
+			}
+			return isExist, nil
+		}
+
+		isExist, err := commitGraphExists()
+		if err != nil {
+			return err
+		}
+		if !isExist {
+			numNeedUpdate++
+			if autofix {
+				if err := gitrepo.WriteCommitGraph(ctx, repo); err != nil {
+					logger.Error("Unable to write commit-graph in %s. Error: %v", repo.FullName(), err)
+					return err
+				}
+				isExist, err := commitGraphExists()
+				if err != nil {
+					return err
+				}
+				if isExist {
+					numWritten++
+					logger.Info("Commit-graph written:    %s", repo.FullName())
+				} else {
+					logger.Warn("No commit-graph written: %s", repo.FullName())
+				}
+			}
+		}
+		return nil
+	}); err != nil {
+		logger.Critical("Unable to checkCommitGraph: %v", err)
+		return err
+	}
+
+	if autofix {
+		logger.Info("Wrote commit-graph files for %d of %d repositories.", numWritten, numRepos)
+	} else {
+		logger.Info("Checked %d repositories, %d without commit-graphs.", numRepos, numNeedUpdate)
+	}
+
+	return nil
+}
+
+func init() {
+	Register(&Check{
+		Title:     "Check if hook files are up-to-date and executable",
+		Name:      "hooks",
+		IsDefault: false,
+		Run:       checkHooks,
+		Priority:  6,
+	})
+	Register(&Check{
+		Title:     "Recalculate Stars number for all user",
+		Name:      "recalculate-stars-number",
+		IsDefault: false,
+		Run:       checkUserStarNum,
+		Priority:  6,
+	})
+	Register(&Check{
+		Title:     "Check git-daemon-export-ok files",
+		Name:      "check-git-daemon-export-ok",
+		IsDefault: false,
+		Run:       checkDaemonExport,
+		Priority:  8,
+	})
+	Register(&Check{
+		Title:     "Check commit-graphs",
+		Name:      "check-commit-graphs",
+		IsDefault: false,
+		Run:       checkCommitGraph,
+		Priority:  9,
+	})
+}

services/doctor/repository.go

@@ -0,0 +1,78 @@
+// Copyright 2023 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package doctor
+
+import (
+	"context"
+
+	"gitea.dev/models/db"
+	"gitea.dev/modules/log"
+	"gitea.dev/modules/storage"
+	repo_service "gitea.dev/services/repository"
+
+	"xorm.io/builder"
+)
+
+func handleDeleteOrphanedRepos(ctx context.Context, logger log.Logger, autofix bool) error {
+	test := &consistencyCheck{
+		Name:         "Repos with no existing owner",
+		Counter:      countOrphanedRepos,
+		Fixer:        deleteOrphanedRepos,
+		FixedMessage: "Deleted all content related to orphaned repos",
+	}
+	return test.Run(ctx, logger, autofix)
+}
+
+// countOrphanedRepos count repository where user of owner_id do not exist
+func countOrphanedRepos(ctx context.Context) (int64, error) {
+	return db.CountOrphanedObjects(ctx, "repository", "user", "repository.owner_id=`user`.id")
+}
+
+// deleteOrphanedRepos delete repository where user of owner_id do not exist
+func deleteOrphanedRepos(ctx context.Context) (int64, error) {
+	if err := storage.Init(); err != nil {
+		return 0, err
+	}
+
+	batchSize := db.MaxBatchInsertSize("repository")
+	e := db.GetEngine(ctx)
+	var deleted int64
+
+	for {
+		select {
+		case <-ctx.Done():
+			return deleted, ctx.Err()
+		default:
+			var ids []int64
+			if err := e.Table("`repository`").
+				Join("LEFT", "`user`", "repository.owner_id=`user`.id").
+				Where(builder.IsNull{"`user`.id"}).
+				Select("`repository`.id").Limit(batchSize).Find(&ids); err != nil {
+				return deleted, err
+			}
+
+			// if we don't get ids we have deleted them all
+			if len(ids) == 0 {
+				return deleted, nil
+			}
+
+			for _, id := range ids {
+				if err := repo_service.DeleteRepositoryDirectly(ctx, id, true); err != nil {
+					return deleted, err
+				}
+				deleted++
+			}
+		}
+	}
+}
+
+func init() {
+	Register(&Check{
+		Title:     "Deleted all content related to orphaned repos",
+		Name:      "delete-orphaned-repos",
+		IsDefault: false,
+		Run:       handleDeleteOrphanedRepos,
+		Priority:  4,
+	})
+}

services/doctor/storage.go

@@ -0,0 +1,270 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package doctor
+
+import (
+	"context"
+	"errors"
+	"io/fs"
+	"strings"
+
+	"gitea.dev/models/git"
+	"gitea.dev/models/packages"
+	"gitea.dev/models/repo"
+	"gitea.dev/models/user"
+	"gitea.dev/modules/base"
+	"gitea.dev/modules/log"
+	packages_module "gitea.dev/modules/packages"
+	"gitea.dev/modules/setting"
+	"gitea.dev/modules/storage"
+	"gitea.dev/modules/util"
+)
+
+type commonStorageCheckOptions struct {
+	storer     storage.ObjectStorage
+	isOrphaned func(path string, obj storage.Object, stat fs.FileInfo) (bool, error)
+	name       string
+}
+
+func commonCheckStorage(logger log.Logger, autofix bool, opts *commonStorageCheckOptions) error {
+	totalCount, orphanedCount := 0, 0
+	totalSize, orphanedSize := int64(0), int64(0)
+
+	var pathsToDelete []string
+	if err := opts.storer.IterateObjects("", func(p string, obj storage.Object) error {
+		defer obj.Close()
+
+		totalCount++
+		stat, err := obj.Stat()
+		if err != nil {
+			return err
+		}
+		totalSize += stat.Size()
+
+		orphaned, err := opts.isOrphaned(p, obj, stat)
+		if err != nil {
+			return err
+		}
+		if orphaned {
+			orphanedCount++
+			orphanedSize += stat.Size()
+			if autofix {
+				pathsToDelete = append(pathsToDelete, p)
+			}
+		}
+		return nil
+	}); err != nil {
+		logger.Error("Error whilst iterating %s storage: %v", opts.name, err)
+		return err
+	}
+
+	if orphanedCount > 0 {
+		if autofix {
+			var deletedNum int
+			for _, p := range pathsToDelete {
+				if err := opts.storer.Delete(p); err != nil {
+					log.Error("Error whilst deleting %s from %s storage: %v", p, opts.name, err)
+				} else {
+					deletedNum++
+				}
+			}
+			logger.Info("Deleted %d/%d orphaned %s(s)", deletedNum, orphanedCount, opts.name)
+		} else {
+			logger.Warn("Found %d/%d (%s/%s) orphaned %s(s)", orphanedCount, totalCount, base.FileSize(orphanedSize), base.FileSize(totalSize), opts.name)
+		}
+	} else {
+		logger.Info("Found %d (%s) %s(s)", totalCount, base.FileSize(totalSize), opts.name)
+	}
+	return nil
+}
+
+type checkStorageOptions struct {
+	All          bool
+	Attachments  bool
+	LFS          bool
+	Avatars      bool
+	RepoAvatars  bool
+	RepoArchives bool
+	Packages     bool
+}
+
+// checkStorage will return a doctor check function to check the requested storage types for "orphaned" stored object/files and optionally delete them
+func checkStorage(opts *checkStorageOptions) func(ctx context.Context, logger log.Logger, autofix bool) error {
+	return func(ctx context.Context, logger log.Logger, autofix bool) error {
+		if err := storage.Init(); err != nil {
+			logger.Error("storage.Init failed: %v", err)
+			return err
+		}
+
+		if opts.Attachments || opts.All {
+			if err := commonCheckStorage(logger, autofix,
+				&commonStorageCheckOptions{
+					storer: storage.Attachments,
+					isOrphaned: func(path string, obj storage.Object, stat fs.FileInfo) (bool, error) {
+						exists, err := repo.ExistAttachmentsByUUID(ctx, stat.Name())
+						return !exists, err
+					},
+					name: "attachment",
+				}); err != nil {
+				return err
+			}
+		}
+
+		if opts.LFS || opts.All {
+			if !setting.LFS.StartServer {
+				logger.Info("LFS isn't enabled (skipped)")
+				return nil
+			}
+			if err := commonCheckStorage(logger, autofix,
+				&commonStorageCheckOptions{
+					storer: storage.LFS,
+					isOrphaned: func(path string, obj storage.Object, stat fs.FileInfo) (bool, error) {
+						// The oid of an LFS stored object is the name but with all the path.Separators removed
+						oid := strings.ReplaceAll(strings.ReplaceAll(path, "\\", ""), "/", "")
+						exists, err := git.ExistsLFSObject(ctx, oid)
+						return !exists, err
+					},
+					name: "LFS file",
+				}); err != nil {
+				return err
+			}
+		}
+
+		if opts.Avatars || opts.All {
+			if err := commonCheckStorage(logger, autofix,
+				&commonStorageCheckOptions{
+					storer: storage.Avatars,
+					isOrphaned: func(path string, obj storage.Object, stat fs.FileInfo) (bool, error) {
+						exists, err := user.ExistsWithAvatarAtStoragePath(ctx, path)
+						return !exists, err
+					},
+					name: "avatar",
+				}); err != nil {
+				return err
+			}
+		}
+
+		if opts.RepoAvatars || opts.All {
+			if err := commonCheckStorage(logger, autofix,
+				&commonStorageCheckOptions{
+					storer: storage.RepoAvatars,
+					isOrphaned: func(path string, obj storage.Object, stat fs.FileInfo) (bool, error) {
+						exists, err := repo.ExistsWithAvatarAtStoragePath(ctx, path)
+						return !exists, err
+					},
+					name: "repo avatar",
+				}); err != nil {
+				return err
+			}
+		}
+
+		if opts.RepoArchives || opts.All {
+			if err := commonCheckStorage(logger, autofix,
+				&commonStorageCheckOptions{
+					storer: storage.RepoArchives,
+					isOrphaned: func(path string, obj storage.Object, stat fs.FileInfo) (bool, error) {
+						exists, err := repo.ExistsRepoArchiverWithStoragePath(ctx, path)
+						if err == nil || errors.Is(err, util.ErrInvalidArgument) {
+							// invalid arguments mean that the object is not a valid repo archiver and it should be removed
+							return !exists, nil
+						}
+						return !exists, err
+					},
+					name: "repo archive",
+				}); err != nil {
+				return err
+			}
+		}
+
+		if opts.Packages || opts.All {
+			if !setting.Packages.Enabled {
+				logger.Info("Packages isn't enabled (skipped)")
+				return nil
+			}
+			if err := commonCheckStorage(logger, autofix,
+				&commonStorageCheckOptions{
+					storer: storage.Packages,
+					isOrphaned: func(path string, obj storage.Object, stat fs.FileInfo) (bool, error) {
+						key, err := packages_module.RelativePathToKey(path)
+						if err != nil {
+							// If there is an error here then the relative path does not match a valid package
+							// Therefore it is orphaned by default
+							return true, nil
+						}
+
+						exists, err := packages.ExistPackageBlobWithSHA(ctx, string(key))
+
+						return !exists, err
+					},
+					name: "package blob",
+				}); err != nil {
+				return err
+			}
+		}
+
+		return nil
+	}
+}
+
+func init() {
+	Register(&Check{
+		Title:                      "Check if there are orphaned storage files",
+		Name:                       "storages",
+		IsDefault:                  false,
+		Run:                        checkStorage(&checkStorageOptions{All: true}),
+		AbortIfFailed:              false,
+		SkipDatabaseInitialization: false,
+		Priority:                   1,
+	})
+
+	Register(&Check{
+		Title:                      "Check if there are orphaned attachments in storage",
+		Name:                       "storage-attachments",
+		IsDefault:                  false,
+		Run:                        checkStorage(&checkStorageOptions{Attachments: true}),
+		AbortIfFailed:              false,
+		SkipDatabaseInitialization: false,
+		Priority:                   1,
+	})
+
+	Register(&Check{
+		Title:                      "Check if there are orphaned lfs files in storage",
+		Name:                       "storage-lfs",
+		IsDefault:                  false,
+		Run:                        checkStorage(&checkStorageOptions{LFS: true}),
+		AbortIfFailed:              false,
+		SkipDatabaseInitialization: false,
+		Priority:                   1,
+	})
+
+	Register(&Check{
+		Title:                      "Check if there are orphaned avatars in storage",
+		Name:                       "storage-avatars",
+		IsDefault:                  false,
+		Run:                        checkStorage(&checkStorageOptions{Avatars: true, RepoAvatars: true}),
+		AbortIfFailed:              false,
+		SkipDatabaseInitialization: false,
+		Priority:                   1,
+	})
+
+	Register(&Check{
+		Title:                      "Check if there are orphaned archives in storage",
+		Name:                       "storage-archives",
+		IsDefault:                  false,
+		Run:                        checkStorage(&checkStorageOptions{RepoArchives: true}),
+		AbortIfFailed:              false,
+		SkipDatabaseInitialization: false,
+		Priority:                   1,
+	})
+
+	Register(&Check{
+		Title:                      "Check if there are orphaned package blobs in storage",
+		Name:                       "storage-packages",
+		IsDefault:                  false,
+		Run:                        checkStorage(&checkStorageOptions{Packages: true}),
+		AbortIfFailed:              false,
+		SkipDatabaseInitialization: false,
+		Priority:                   1,
+	})
+}