Cache statistics and provide estimation methods

docs/content/doc/advanced/config-cheat-sheet.en-us.md

@@ -197,6 +197,7 @@ The following configuration set `Content-Type: application/vnd.android.package-a
 - `REPO_PAGING_NUM`: **50**: Number of repos that are shown in one page.
 - `NOTICE_PAGING_NUM`: **25**: Number of notices that are shown in one page.
 - `ORG_PAGING_NUM`: **50**: Number of organizations that are shown in one page.
+- `STATISTICS_TTL`: **5m**: Cache summary statistics for this period of time.
 
 ### UI - Metadata (`ui.meta`)
 
@@ -975,6 +976,7 @@ Default templates for project boards:
 - `ENABLED_ISSUE_BY_LABEL`: **false**: Enable issue by label metrics with format `gitea_issues_by_label{label="bug"} 2`.
 - `ENABLED_ISSUE_BY_REPOSITORY`: **false**: Enable issue by repository metrics with format `gitea_issues_by_repository{repository="org/repo"} 5`.
 - `TOKEN`: **\<empty\>**: You need to specify the token, if you want to include in the authorization the metrics . The same token need to be used in prometheus parameters `bearer_token` or `bearer_token_file`.
+- `STATISTICS_TTL`: **5m**: Cache summary statistics for this period of time.
 
 ## API (`api`)
 

models/statistic.go

@@ -5,6 +5,9 @@
 package models
 
 import (
+	"context"
+	"time"
+
 	asymkey_model "code.gitea.io/gitea/models/asymkey"
 	"code.gitea.io/gitea/models/auth"
 	"code.gitea.io/gitea/models/db"
@@ -32,6 +35,7 @@ type Statistic struct {
 		IssueByLabel      []IssueByLabelCount
 		IssueByRepository []IssueByRepositoryCount
 	}
+	Time time.Time
 }
 
 // IssueByLabelCount contains the number of issue group by label
@@ -48,23 +52,24 @@ type IssueByRepositoryCount struct {
 }
 
 // GetStatistic returns the database statistics
-func GetStatistic() (stats Statistic) {
-	e := db.GetEngine(db.DefaultContext)
+func GetStatistic(ctx context.Context, metrics bool) (stats Statistic) {
+	e := db.GetEngine(ctx)
+
 	stats.Counter.User = user_model.CountUsers(nil)
 	stats.Counter.Org, _ = organization.CountOrgs(organization.FindOrgOptions{IncludePrivate: true})
 	stats.Counter.PublicKey, _ = e.Count(new(asymkey_model.PublicKey))
 	stats.Counter.Repo, _ = repo_model.CountRepositories(db.DefaultContext, repo_model.CountRepositoryOptions{})
 	stats.Counter.Watch, _ = e.Count(new(repo_model.Watch))
 	stats.Counter.Star, _ = e.Count(new(repo_model.Star))
-	stats.Counter.Action, _ = db.EstimateCount(db.DefaultContext, new(Action))
-	stats.Counter.Access, _ = e.Count(new(access_model.Access))
+	stats.Counter.Action, _ = db.EstimateCount(ctx, new(Action))
+	stats.Counter.Access, _ = db.EstimateCount(ctx, new(access_model.Access))
 
 	type IssueCount struct {
 		Count    int64
 		IsClosed bool
 	}
 
-	if setting.Metrics.EnabledIssueByLabel {
+	if metrics && setting.Metrics.EnabledIssueByLabel {
 		stats.Counter.IssueByLabel = []IssueByLabelCount{}
 
 		_ = e.Select("COUNT(*) AS count, l.name AS label").
@@ -74,7 +79,7 @@ func GetStatistic() (stats Statistic) {
 			Find(&stats.Counter.IssueByLabel)
 	}
 
-	if setting.Metrics.EnabledIssueByRepository {
+	if metrics && setting.Metrics.EnabledIssueByRepository {
 		stats.Counter.IssueByRepository = []IssueByRepositoryCount{}
 
 		_ = e.Select("COUNT(*) AS count, r.owner_name, r.name AS repository").
@@ -97,19 +102,20 @@ func GetStatistic() (stats Statistic) {
 
 	stats.Counter.Issue = stats.Counter.IssueClosed + stats.Counter.IssueOpen
 
-	stats.Counter.Comment, _ = e.Count(new(Comment))
-	stats.Counter.Oauth = 0
+	stats.Counter.Comment, _ = db.EstimateCount(ctx, new(Comment))
 	stats.Counter.Follow, _ = e.Count(new(user_model.Follow))
 	stats.Counter.Mirror, _ = e.Count(new(repo_model.Mirror))
 	stats.Counter.Release, _ = e.Count(new(Release))
-	stats.Counter.AuthSource = auth.CountSources()
 	stats.Counter.Webhook, _ = e.Count(new(webhook.Webhook))
 	stats.Counter.Milestone, _ = e.Count(new(issues_model.Milestone))
 	stats.Counter.Label, _ = e.Count(new(Label))
-	stats.Counter.HookTask, _ = e.Count(new(webhook.HookTask))
+	stats.Counter.HookTask, _ = db.EstimateCount(ctx, new(webhook.HookTask))
 	stats.Counter.Team, _ = e.Count(new(organization.Team))
-	stats.Counter.Attachment, _ = e.Count(new(repo_model.Attachment))
+	stats.Counter.Attachment, _ = db.EstimateCount(ctx, new(repo_model.Attachment))
 	stats.Counter.Project, _ = e.Count(new(project_model.Project))
 	stats.Counter.ProjectBoard, _ = e.Count(new(project_model.Board))
+	stats.Counter.Oauth = 0
+	stats.Counter.AuthSource = auth.CountSources()
+	stats.Time = time.Now()
 	return
 }

modules/metrics/collector.go

@@ -6,6 +6,7 @@ package metrics
 
 import (
 	"code.gitea.io/gitea/models"
+	"code.gitea.io/gitea/modules/setting"
 
 	"github.com/prometheus/client_golang/prometheus"
 )
@@ -43,6 +44,8 @@ type Collector struct {
 	Users              *prometheus.Desc
 	Watches            *prometheus.Desc
 	Webhooks           *prometheus.Desc
+
+	StatisticsTime *prometheus.Desc
 }
 
 // NewCollector returns a new Collector with all prometheus.Desc initialized
@@ -225,152 +228,156 @@ func (c Collector) Describe(ch chan<- *prometheus.Desc) {
 
 // Collect returns the metrics with values
 func (c Collector) Collect(ch chan<- prometheus.Metric) {
-	stats := models.GetStatistic()
+	stats := <-GetStatistic(setting.Metrics.StatisticTTL, true)
+	if stats == nil {
+		// This will happen if the statistics generation was cancelled midway through
+		stats = &models.Statistic{}
+	}
 
-	ch <- prometheus.MustNewConstMetric(
+	ch <- prometheus.NewMetricWithTimestamp(stats.Time, prometheus.MustNewConstMetric(
 		c.Accesses,
 		prometheus.GaugeValue,
 		float64(stats.Counter.Access),
-	)
-	ch <- prometheus.MustNewConstMetric(
+	))
+	ch <- prometheus.NewMetricWithTimestamp(stats.Time, prometheus.MustNewConstMetric(
 		c.Actions,
 		prometheus.GaugeValue,
 		float64(stats.Counter.Action),
-	)
-	ch <- prometheus.MustNewConstMetric(
+	))
+	ch <- prometheus.NewMetricWithTimestamp(stats.Time, prometheus.MustNewConstMetric(
 		c.Attachments,
 		prometheus.GaugeValue,
 		float64(stats.Counter.Attachment),
-	)
-	ch <- prometheus.MustNewConstMetric(
+	))
+	ch <- prometheus.NewMetricWithTimestamp(stats.Time, prometheus.MustNewConstMetric(
 		c.Comments,
 		prometheus.GaugeValue,
 		float64(stats.Counter.Comment),
-	)
-	ch <- prometheus.MustNewConstMetric(
+	))
+	ch <- prometheus.NewMetricWithTimestamp(stats.Time, prometheus.MustNewConstMetric(
 		c.Follows,
 		prometheus.GaugeValue,
 		float64(stats.Counter.Follow),
-	)
-	ch <- prometheus.MustNewConstMetric(
+	))
+	ch <- prometheus.NewMetricWithTimestamp(stats.Time, prometheus.MustNewConstMetric(
 		c.HookTasks,
 		prometheus.GaugeValue,
 		float64(stats.Counter.HookTask),
-	)
-	ch <- prometheus.MustNewConstMetric(
+	))
+	ch <- prometheus.NewMetricWithTimestamp(stats.Time, prometheus.MustNewConstMetric(
 		c.Issues,
 		prometheus.GaugeValue,
 		float64(stats.Counter.Issue),
-	)
+	))
 	for _, il := range stats.Counter.IssueByLabel {
-		ch <- prometheus.MustNewConstMetric(
+		ch <- prometheus.NewMetricWithTimestamp(stats.Time, prometheus.MustNewConstMetric(
 			c.IssuesByLabel,
 			prometheus.GaugeValue,
 			float64(il.Count),
 			il.Label,
-		)
+		))
 	}
 	for _, ir := range stats.Counter.IssueByRepository {
-		ch <- prometheus.MustNewConstMetric(
+		ch <- prometheus.NewMetricWithTimestamp(stats.Time, prometheus.MustNewConstMetric(
 			c.IssuesByRepository,
 			prometheus.GaugeValue,
 			float64(ir.Count),
 			ir.OwnerName+"/"+ir.Repository,
-		)
+		))
 	}
-	ch <- prometheus.MustNewConstMetric(
+	ch <- prometheus.NewMetricWithTimestamp(stats.Time, prometheus.MustNewConstMetric(
 		c.IssuesClosed,
 		prometheus.GaugeValue,
 		float64(stats.Counter.IssueClosed),
-	)
-	ch <- prometheus.MustNewConstMetric(
+	))
+	ch <- prometheus.NewMetricWithTimestamp(stats.Time, prometheus.MustNewConstMetric(
 		c.IssuesOpen,
 		prometheus.GaugeValue,
 		float64(stats.Counter.IssueOpen),
-	)
-	ch <- prometheus.MustNewConstMetric(
+	))
+	ch <- prometheus.NewMetricWithTimestamp(stats.Time, prometheus.MustNewConstMetric(
 		c.Labels,
 		prometheus.GaugeValue,
 		float64(stats.Counter.Label),
-	)
-	ch <- prometheus.MustNewConstMetric(
+	))
+	ch <- prometheus.NewMetricWithTimestamp(stats.Time, prometheus.MustNewConstMetric(
 		c.LoginSources,
 		prometheus.GaugeValue,
 		float64(stats.Counter.AuthSource),
-	)
-	ch <- prometheus.MustNewConstMetric(
+	))
+	ch <- prometheus.NewMetricWithTimestamp(stats.Time, prometheus.MustNewConstMetric(
 		c.Milestones,
 		prometheus.GaugeValue,
 		float64(stats.Counter.Milestone),
-	)
-	ch <- prometheus.MustNewConstMetric(
+	))
+	ch <- prometheus.NewMetricWithTimestamp(stats.Time, prometheus.MustNewConstMetric(
 		c.Mirrors,
 		prometheus.GaugeValue,
 		float64(stats.Counter.Mirror),
-	)
-	ch <- prometheus.MustNewConstMetric(
+	))
+	ch <- prometheus.NewMetricWithTimestamp(stats.Time, prometheus.MustNewConstMetric(
 		c.Oauths,
 		prometheus.GaugeValue,
 		float64(stats.Counter.Oauth),
-	)
-	ch <- prometheus.MustNewConstMetric(
+	))
+	ch <- prometheus.NewMetricWithTimestamp(stats.Time, prometheus.MustNewConstMetric(
 		c.Organizations,
 		prometheus.GaugeValue,
 		float64(stats.Counter.Org),
-	)
-	ch <- prometheus.MustNewConstMetric(
+	))
+	ch <- prometheus.NewMetricWithTimestamp(stats.Time, prometheus.MustNewConstMetric(
 		c.Projects,
 		prometheus.GaugeValue,
 		float64(stats.Counter.Project),
-	)
-	ch <- prometheus.MustNewConstMetric(
+	))
+	ch <- prometheus.NewMetricWithTimestamp(stats.Time, prometheus.MustNewConstMetric(
 		c.ProjectBoards,
 		prometheus.GaugeValue,
 		float64(stats.Counter.ProjectBoard),
-	)
-	ch <- prometheus.MustNewConstMetric(
+	))
+	ch <- prometheus.NewMetricWithTimestamp(stats.Time, prometheus.MustNewConstMetric(
 		c.PublicKeys,
 		prometheus.GaugeValue,
 		float64(stats.Counter.PublicKey),
-	)
-	ch <- prometheus.MustNewConstMetric(
+	))
+	ch <- prometheus.NewMetricWithTimestamp(stats.Time, prometheus.MustNewConstMetric(
 		c.Releases,
 		prometheus.GaugeValue,
 		float64(stats.Counter.Release),
-	)
-	ch <- prometheus.MustNewConstMetric(
+	))
+	ch <- prometheus.NewMetricWithTimestamp(stats.Time, prometheus.MustNewConstMetric(
 		c.Repositories,
 		prometheus.GaugeValue,
 		float64(stats.Counter.Repo),
-	)
-	ch <- prometheus.MustNewConstMetric(
+	))
+	ch <- prometheus.NewMetricWithTimestamp(stats.Time, prometheus.MustNewConstMetric(
 		c.Stars,
 		prometheus.GaugeValue,
 		float64(stats.Counter.Star),
-	)
-	ch <- prometheus.MustNewConstMetric(
+	))
+	ch <- prometheus.NewMetricWithTimestamp(stats.Time, prometheus.MustNewConstMetric(
 		c.Teams,
 		prometheus.GaugeValue,
 		float64(stats.Counter.Team),
-	)
-	ch <- prometheus.MustNewConstMetric(
+	))
+	ch <- prometheus.NewMetricWithTimestamp(stats.Time, prometheus.MustNewConstMetric(
 		c.UpdateTasks,
 		prometheus.GaugeValue,
 		float64(stats.Counter.UpdateTask),
-	)
-	ch <- prometheus.MustNewConstMetric(
+	))
+	ch <- prometheus.NewMetricWithTimestamp(stats.Time, prometheus.MustNewConstMetric(
 		c.Users,
 		prometheus.GaugeValue,
 		float64(stats.Counter.User),
-	)
-	ch <- prometheus.MustNewConstMetric(
+	))
+	ch <- prometheus.NewMetricWithTimestamp(stats.Time, prometheus.MustNewConstMetric(
 		c.Watches,
 		prometheus.GaugeValue,
 		float64(stats.Counter.Watch),
-	)
-	ch <- prometheus.MustNewConstMetric(
+	))
+	ch <- prometheus.NewMetricWithTimestamp(stats.Time, prometheus.MustNewConstMetric(
 		c.Webhooks,
 		prometheus.GaugeValue,
 		float64(stats.Counter.Webhook),
-	)
+	))
 }