fix: Fix issue with requests totals

Problem is that we join with PackageBase, thus we are missing
requests for packages that were deleted.

Fixes: #483
Signed-off-by: moson <moson@archlinux.org>

aurweb/routers/html.py

@@ -16,11 +16,10 @@ from sqlalchemy import case, or_
 
 import aurweb.config
 import aurweb.models.package_request
-from aurweb import aur_logging, cookies, db, models, time, util
+from aurweb import aur_logging, cookies, db, models, statistics, time, util
 from aurweb.exceptions import handle_form_exceptions
 from aurweb.models.package_request import PENDING_ID
 from aurweb.packages.util import query_notified, query_voted, updated_packages
-from aurweb.statistics import Statistics, update_prometheus_metrics
 from aurweb.templates import make_context, render_template
 
 logger = aur_logging.get_logger(__name__)
@@ -89,9 +88,9 @@ async def index(request: Request):
     cache_expire = aurweb.config.getint("cache", "expiry_time_statistics", 300)
 
     # Package statistics.
-    stats = Statistics(cache_expire)
-    for counter in stats.HOMEPAGE_COUNTERS:
-        context[counter] = stats.get_count(counter)
+    counts = statistics.get_homepage_counts()
+    for k in counts:
+        context[k] = counts[k]
 
     # Get the 15 most recently updated packages.
     context["package_updates"] = updated_packages(15, cache_expire)
@@ -213,7 +212,7 @@ async def metrics(request: Request):
         )
 
     # update prometheus gauges for packages and users
-    update_prometheus_metrics()
+    statistics.update_prometheus_metrics()
 
     registry = CollectorRegistry()
     multiprocess.MultiProcessCollector(registry)

aurweb/routers/requests.py

@@ -16,6 +16,7 @@ from aurweb.models.package_request import (
 )
 from aurweb.requests.util import get_pkgreq_by_id
 from aurweb.scripts import notify
+from aurweb.statistics import get_request_counts
 from aurweb.templates import make_context, render_template
 
 FILTER_PARAMS = {
@@ -31,7 +32,7 @@ router = APIRouter()
 
 @router.get("/requests")
 @requires_auth
-async def requests(
+async def requests(  # noqa: C901
     request: Request,
     O: int = Query(default=defaults.O),
     PP: int = Query(default=defaults.PP),
@@ -74,18 +75,11 @@ async def requests(
         .join(User, PackageRequest.UsersID == User.ID, isouter=True)
         .join(Maintainer, PackageBase.MaintainerUID == Maintainer.ID, isouter=True)
     )
-    # query = db.query(PackageRequest).join(User)
 
     # Requests statistics
-    context["total_requests"] = query.count()
-    pending_count = 0 + query.filter(PackageRequest.Status == PENDING_ID).count()
-    context["pending_requests"] = pending_count
-    closed_count = 0 + query.filter(PackageRequest.Status == CLOSED_ID).count()
-    context["closed_requests"] = closed_count
-    accepted_count = 0 + query.filter(PackageRequest.Status == ACCEPTED_ID).count()
-    context["accepted_requests"] = accepted_count
-    rejected_count = 0 + query.filter(PackageRequest.Status == REJECTED_ID).count()
-    context["rejected_requests"] = rejected_count
+    counts = get_request_counts()
+    for k in counts:
+        context[k] = counts[k]
 
     # Apply status filters
     in_filters = []

aurweb/statistics.py

@@ -1,31 +1,46 @@
 from aurweb import config, db, time
 from aurweb.cache import db_count_cache
-from aurweb.models import PackageBase, User
+from aurweb.models import PackageBase, PackageRequest, User
 from aurweb.models.account_type import TRUSTED_USER_AND_DEV_ID, TRUSTED_USER_ID, USER_ID
+from aurweb.models.package_request import (
+    ACCEPTED_ID,
+    CLOSED_ID,
+    PENDING_ID,
+    REJECTED_ID,
+)
 from aurweb.prometheus import PACKAGES, USERS
 
+cache_expire = config.getint("cache", "expiry_time_statistics", 300)
+
+HOMEPAGE_COUNTERS = [
+    "package_count",
+    "orphan_count",
+    "seven_days_old_added",
+    "seven_days_old_updated",
+    "year_old_updated",
+    "never_updated",
+    "user_count",
+    "trusted_user_count",
+]
+REQUEST_COUNTERS = [
+    "total_requests",
+    "pending_requests",
+    "closed_requests",
+    "accepted_requests",
+    "rejected_requests",
+]
+PROMETHEUS_USER_COUNTERS = [
+    ("trusted_user_count", "tu"),
+    ("regular_user_count", "user"),
+]
+PROMETHEUS_PACKAGE_COUNTERS = [
+    ("orphan_count", "orphan"),
+    ("never_updated", "not_updated"),
+    ("updated_packages", "updated"),
+]
+
 
 class Statistics:
-    HOMEPAGE_COUNTERS = [
-        "package_count",
-        "orphan_count",
-        "seven_days_old_added",
-        "seven_days_old_updated",
-        "year_old_updated",
-        "never_updated",
-        "user_count",
-        "trusted_user_count",
-    ]
-    PROMETHEUS_USER_COUNTERS = [
-        ("trusted_user_count", "tu"),
-        ("regular_user_count", "user"),
-    ]
-    PROMETHEUS_PACKAGE_COUNTERS = [
-        ("orphan_count", "orphan"),
-        ("never_updated", "not_updated"),
-        ("updated_packages", "updated"),
-    ]
-
     seven_days = 86400 * 7
     one_hour = 3600
     year = seven_days * 52
@@ -35,15 +50,18 @@ class Statistics:
         self.now = time.utcnow()
         self.seven_days_ago = self.now - self.seven_days
         self.year_ago = self.now - self.year
+
         self.user_query = db.query(User)
         self.bases_query = db.query(PackageBase)
         self.updated_query = db.query(PackageBase).filter(
             PackageBase.ModifiedTS - PackageBase.SubmittedTS >= self.one_hour
         )
+        self.request_query = db.query(PackageRequest)
 
     def get_count(self, counter: str) -> int:
         query = None
         match counter:
+            # Packages
             case "package_count":
                 query = self.bases_query
             case "orphan_count":
@@ -69,6 +87,7 @@ class Statistics:
                     PackageBase.ModifiedTS - PackageBase.SubmittedTS > self.one_hour,
                     ~PackageBase.MaintainerUID.is_(None),
                 )
+            # Users
             case "user_count":
                 query = self.user_query
             case "trusted_user_count":
@@ -82,6 +101,18 @@ class Statistics:
                 )
             case "regular_user_count":
                 query = self.user_query.filter(User.AccountTypeID == USER_ID)
+
+            # Requests
+            case "total_requests":
+                query = self.request_query
+            case "pending_requests":
+                query = self.request_query.filter(PackageRequest.Status == PENDING_ID)
+            case "closed_requests":
+                query = self.request_query.filter(PackageRequest.Status == CLOSED_ID)
+            case "accepted_requests":
+                query = self.request_query.filter(PackageRequest.Status == ACCEPTED_ID)
+            case "rejected_requests":
+                query = self.request_query.filter(PackageRequest.Status == REJECTED_ID)
             case _:
                 return -1
 
@@ -89,14 +120,30 @@ class Statistics:
 
 
 def update_prometheus_metrics():
-    cache_expire = config.getint("cache", "expiry_time_statistics", 300)
     stats = Statistics(cache_expire)
     # Users gauge
-    for counter, utype in stats.PROMETHEUS_USER_COUNTERS:
+    for counter, utype in PROMETHEUS_USER_COUNTERS:
         count = stats.get_count(counter)
         USERS.labels(utype).set(count)
 
     # Packages gauge
-    for counter, state in stats.PROMETHEUS_PACKAGE_COUNTERS:
+    for counter, state in PROMETHEUS_PACKAGE_COUNTERS:
         count = stats.get_count(counter)
         PACKAGES.labels(state).set(count)
+
+
+def _get_counts(counters: list[str]) -> dict[str, int]:
+    stats = Statistics(cache_expire)
+    result = dict()
+    for counter in counters:
+        result[counter] = stats.get_count(counter)
+
+    return result
+
+
+def get_homepage_counts() -> dict[str, int]:
+    return _get_counts(HOMEPAGE_COUNTERS)
+
+
+def get_request_counts() -> dict[str, int]:
+    return _get_counts(REQUEST_COUNTERS)

test/test_statistics.py

@@ -2,9 +2,15 @@ import pytest
 from prometheus_client import REGISTRY, generate_latest
 
 from aurweb import cache, db, time
+from aurweb.models import Package, PackageBase, PackageRequest
 from aurweb.models.account_type import TRUSTED_USER_ID, USER_ID
-from aurweb.models.package import Package
-from aurweb.models.package_base import PackageBase
+from aurweb.models.package_request import (
+    ACCEPTED_ID,
+    CLOSED_ID,
+    PENDING_ID,
+    REJECTED_ID,
+)
+from aurweb.models.request_type import DELETION_ID, ORPHAN_ID
 from aurweb.models.user import User
 from aurweb.statistics import Statistics, update_prometheus_metrics
 
@@ -45,19 +51,36 @@ def test_data():
                 ModifiedTS=now,
             )
             db.create(Package, PackageBase=pkgbase, Name=pkgbase.Name)
+            pkgreq = db.create(
+                PackageRequest,
+                ReqTypeID=ORPHAN_ID,
+                User=user,
+                PackageBase=pkgbase,
+                PackageBaseName=pkgbase.Name,
+                RequestTS=now,
+                Comments=str(),
+                ClosureComment=str(),
+            )
 
             # Modify some data to get some variances for our counters
             if i == 1:
                 user.AccountTypeID = TRUSTED_USER_ID
                 pkgbase.Maintainer = None
                 pkgbase.SubmittedTS = now
+                pkgreq.Status = PENDING_ID
+                pkgreq.ReqTypeID = DELETION_ID
 
             if i == 2:
                 pkgbase.SubmittedTS = older
+                pkgreq.Status = ACCEPTED_ID
 
             if i == 3:
                 pkgbase.SubmittedTS = older
                 pkgbase.ModifiedTS = old
+                pkgreq.Status = CLOSED_ID
+
+            if i == 4:
+                pkgreq.Status = REJECTED_ID
     yield
 
 
@@ -79,6 +102,11 @@ def stats() -> Statistics:
         ("trusted_user_count", 1),
         ("regular_user_count", 9),
         ("updated_packages", 9),
+        ("total_requests", 10),
+        ("pending_requests", 7),
+        ("closed_requests", 1),
+        ("accepted_requests", 1),
+        ("rejected_requests", 1),
         ("nonsense", -1),
     ],
 )