From e45878a058071e59f0f08eb3dca597f560448298 Mon Sep 17 00:00:00 2001 From: moson Date: Sun, 23 Jul 2023 18:53:58 +0200 Subject: [PATCH] fix: Fix issue with requests totals Problem is that we join with PackageBase, thus we are missing requests for packages that were deleted. Fixes: #483 Signed-off-by: moson --- aurweb/routers/html.py | 11 ++--- aurweb/routers/requests.py | 16 ++----- aurweb/statistics.py | 95 ++++++++++++++++++++++++++++---------- test/test_statistics.py | 32 ++++++++++++- 4 files changed, 111 insertions(+), 43 deletions(-) diff --git a/aurweb/routers/html.py b/aurweb/routers/html.py index 2ec497bd..63cc3bb8 100644 --- a/aurweb/routers/html.py +++ b/aurweb/routers/html.py @@ -16,11 +16,10 @@ from sqlalchemy import case, or_ import aurweb.config import aurweb.models.package_request -from aurweb import aur_logging, cookies, db, models, time, util +from aurweb import aur_logging, cookies, db, models, statistics, time, util from aurweb.exceptions import handle_form_exceptions from aurweb.models.package_request import PENDING_ID from aurweb.packages.util import query_notified, query_voted, updated_packages -from aurweb.statistics import Statistics, update_prometheus_metrics from aurweb.templates import make_context, render_template logger = aur_logging.get_logger(__name__) @@ -89,9 +88,9 @@ async def index(request: Request): cache_expire = aurweb.config.getint("cache", "expiry_time_statistics", 300) # Package statistics. - stats = Statistics(cache_expire) - for counter in stats.HOMEPAGE_COUNTERS: - context[counter] = stats.get_count(counter) + counts = statistics.get_homepage_counts() + for k in counts: + context[k] = counts[k] # Get the 15 most recently updated packages. context["package_updates"] = updated_packages(15, cache_expire) @@ -213,7 +212,7 @@ async def metrics(request: Request): ) # update prometheus gauges for packages and users - update_prometheus_metrics() + statistics.update_prometheus_metrics() registry = CollectorRegistry() multiprocess.MultiProcessCollector(registry) diff --git a/aurweb/routers/requests.py b/aurweb/routers/requests.py index 4cfda269..a67419fe 100644 --- a/aurweb/routers/requests.py +++ b/aurweb/routers/requests.py @@ -16,6 +16,7 @@ from aurweb.models.package_request import ( ) from aurweb.requests.util import get_pkgreq_by_id from aurweb.scripts import notify +from aurweb.statistics import get_request_counts from aurweb.templates import make_context, render_template FILTER_PARAMS = { @@ -31,7 +32,7 @@ router = APIRouter() @router.get("/requests") @requires_auth -async def requests( +async def requests( # noqa: C901 request: Request, O: int = Query(default=defaults.O), PP: int = Query(default=defaults.PP), @@ -74,18 +75,11 @@ async def requests( .join(User, PackageRequest.UsersID == User.ID, isouter=True) .join(Maintainer, PackageBase.MaintainerUID == Maintainer.ID, isouter=True) ) - # query = db.query(PackageRequest).join(User) # Requests statistics - context["total_requests"] = query.count() - pending_count = 0 + query.filter(PackageRequest.Status == PENDING_ID).count() - context["pending_requests"] = pending_count - closed_count = 0 + query.filter(PackageRequest.Status == CLOSED_ID).count() - context["closed_requests"] = closed_count - accepted_count = 0 + query.filter(PackageRequest.Status == ACCEPTED_ID).count() - context["accepted_requests"] = accepted_count - rejected_count = 0 + query.filter(PackageRequest.Status == REJECTED_ID).count() - context["rejected_requests"] = rejected_count + counts = get_request_counts() + for k in counts: + context[k] = counts[k] # Apply status filters in_filters = [] diff --git a/aurweb/statistics.py b/aurweb/statistics.py index 6e9dbe1f..3c1298b7 100644 --- a/aurweb/statistics.py +++ b/aurweb/statistics.py @@ -1,31 +1,46 @@ from aurweb import config, db, time from aurweb.cache import db_count_cache -from aurweb.models import PackageBase, User +from aurweb.models import PackageBase, PackageRequest, User from aurweb.models.account_type import TRUSTED_USER_AND_DEV_ID, TRUSTED_USER_ID, USER_ID +from aurweb.models.package_request import ( + ACCEPTED_ID, + CLOSED_ID, + PENDING_ID, + REJECTED_ID, +) from aurweb.prometheus import PACKAGES, USERS +cache_expire = config.getint("cache", "expiry_time_statistics", 300) + +HOMEPAGE_COUNTERS = [ + "package_count", + "orphan_count", + "seven_days_old_added", + "seven_days_old_updated", + "year_old_updated", + "never_updated", + "user_count", + "trusted_user_count", +] +REQUEST_COUNTERS = [ + "total_requests", + "pending_requests", + "closed_requests", + "accepted_requests", + "rejected_requests", +] +PROMETHEUS_USER_COUNTERS = [ + ("trusted_user_count", "tu"), + ("regular_user_count", "user"), +] +PROMETHEUS_PACKAGE_COUNTERS = [ + ("orphan_count", "orphan"), + ("never_updated", "not_updated"), + ("updated_packages", "updated"), +] + class Statistics: - HOMEPAGE_COUNTERS = [ - "package_count", - "orphan_count", - "seven_days_old_added", - "seven_days_old_updated", - "year_old_updated", - "never_updated", - "user_count", - "trusted_user_count", - ] - PROMETHEUS_USER_COUNTERS = [ - ("trusted_user_count", "tu"), - ("regular_user_count", "user"), - ] - PROMETHEUS_PACKAGE_COUNTERS = [ - ("orphan_count", "orphan"), - ("never_updated", "not_updated"), - ("updated_packages", "updated"), - ] - seven_days = 86400 * 7 one_hour = 3600 year = seven_days * 52 @@ -35,15 +50,18 @@ class Statistics: self.now = time.utcnow() self.seven_days_ago = self.now - self.seven_days self.year_ago = self.now - self.year + self.user_query = db.query(User) self.bases_query = db.query(PackageBase) self.updated_query = db.query(PackageBase).filter( PackageBase.ModifiedTS - PackageBase.SubmittedTS >= self.one_hour ) + self.request_query = db.query(PackageRequest) def get_count(self, counter: str) -> int: query = None match counter: + # Packages case "package_count": query = self.bases_query case "orphan_count": @@ -69,6 +87,7 @@ class Statistics: PackageBase.ModifiedTS - PackageBase.SubmittedTS > self.one_hour, ~PackageBase.MaintainerUID.is_(None), ) + # Users case "user_count": query = self.user_query case "trusted_user_count": @@ -82,6 +101,18 @@ class Statistics: ) case "regular_user_count": query = self.user_query.filter(User.AccountTypeID == USER_ID) + + # Requests + case "total_requests": + query = self.request_query + case "pending_requests": + query = self.request_query.filter(PackageRequest.Status == PENDING_ID) + case "closed_requests": + query = self.request_query.filter(PackageRequest.Status == CLOSED_ID) + case "accepted_requests": + query = self.request_query.filter(PackageRequest.Status == ACCEPTED_ID) + case "rejected_requests": + query = self.request_query.filter(PackageRequest.Status == REJECTED_ID) case _: return -1 @@ -89,14 +120,30 @@ class Statistics: def update_prometheus_metrics(): - cache_expire = config.getint("cache", "expiry_time_statistics", 300) stats = Statistics(cache_expire) # Users gauge - for counter, utype in stats.PROMETHEUS_USER_COUNTERS: + for counter, utype in PROMETHEUS_USER_COUNTERS: count = stats.get_count(counter) USERS.labels(utype).set(count) # Packages gauge - for counter, state in stats.PROMETHEUS_PACKAGE_COUNTERS: + for counter, state in PROMETHEUS_PACKAGE_COUNTERS: count = stats.get_count(counter) PACKAGES.labels(state).set(count) + + +def _get_counts(counters: list[str]) -> dict[str, int]: + stats = Statistics(cache_expire) + result = dict() + for counter in counters: + result[counter] = stats.get_count(counter) + + return result + + +def get_homepage_counts() -> dict[str, int]: + return _get_counts(HOMEPAGE_COUNTERS) + + +def get_request_counts() -> dict[str, int]: + return _get_counts(REQUEST_COUNTERS) diff --git a/test/test_statistics.py b/test/test_statistics.py index dda7b357..a6a814c5 100644 --- a/test/test_statistics.py +++ b/test/test_statistics.py @@ -2,9 +2,15 @@ import pytest from prometheus_client import REGISTRY, generate_latest from aurweb import cache, db, time +from aurweb.models import Package, PackageBase, PackageRequest from aurweb.models.account_type import TRUSTED_USER_ID, USER_ID -from aurweb.models.package import Package -from aurweb.models.package_base import PackageBase +from aurweb.models.package_request import ( + ACCEPTED_ID, + CLOSED_ID, + PENDING_ID, + REJECTED_ID, +) +from aurweb.models.request_type import DELETION_ID, ORPHAN_ID from aurweb.models.user import User from aurweb.statistics import Statistics, update_prometheus_metrics @@ -45,19 +51,36 @@ def test_data(): ModifiedTS=now, ) db.create(Package, PackageBase=pkgbase, Name=pkgbase.Name) + pkgreq = db.create( + PackageRequest, + ReqTypeID=ORPHAN_ID, + User=user, + PackageBase=pkgbase, + PackageBaseName=pkgbase.Name, + RequestTS=now, + Comments=str(), + ClosureComment=str(), + ) # Modify some data to get some variances for our counters if i == 1: user.AccountTypeID = TRUSTED_USER_ID pkgbase.Maintainer = None pkgbase.SubmittedTS = now + pkgreq.Status = PENDING_ID + pkgreq.ReqTypeID = DELETION_ID if i == 2: pkgbase.SubmittedTS = older + pkgreq.Status = ACCEPTED_ID if i == 3: pkgbase.SubmittedTS = older pkgbase.ModifiedTS = old + pkgreq.Status = CLOSED_ID + + if i == 4: + pkgreq.Status = REJECTED_ID yield @@ -79,6 +102,11 @@ def stats() -> Statistics: ("trusted_user_count", 1), ("regular_user_count", 9), ("updated_packages", 9), + ("total_requests", 10), + ("pending_requests", 7), + ("closed_requests", 1), + ("accepted_requests", 1), + ("rejected_requests", 1), ("nonsense", -1), ], )