From 44c158b8c2667ace8e44d2cac3b7aed4efcc1464 Mon Sep 17 00:00:00 2001 From: moson Date: Sat, 22 Jul 2023 16:31:50 +0200 Subject: [PATCH] feat: Implement statistics class & additional metrics The new module/class helps us constructing queries and count records to expose various statistics on the homepage. We also utilize for some new prometheus metrics (package and user gauges). Record counts are being cached with Redis. Signed-off-by: moson --- aurweb/cache.py | 11 ++-- aurweb/prometheus.py | 18 ++++++- aurweb/routers/html.py | 72 ++++---------------------- aurweb/routers/packages.py | 6 +-- aurweb/statistics.py | 102 +++++++++++++++++++++++++++++++++++++ test/test_cache.py | 18 +++---- test/test_metrics.py | 5 +- 7 files changed, 143 insertions(+), 89 deletions(-) create mode 100644 aurweb/statistics.py diff --git a/aurweb/cache.py b/aurweb/cache.py index fe1e5f1d..bb374e57 100644 --- a/aurweb/cache.py +++ b/aurweb/cache.py @@ -1,20 +1,15 @@ import pickle -from prometheus_client import Counter from sqlalchemy import orm from aurweb import config from aurweb.aur_redis import redis_connection +from aurweb.prometheus import SEARCH_REQUESTS _redis = redis_connection() -# Prometheus metrics -SEARCH_REQUESTS = Counter( - "search_requests", "Number of search requests by cache hit/miss", ["cache"] -) - -async def db_count_cache(key: str, query: orm.Query, expire: int = None) -> int: +def db_count_cache(key: str, query: orm.Query, expire: int = None) -> int: """Store and retrieve a query.count() via redis cache. :param key: Redis key @@ -30,7 +25,7 @@ async def db_count_cache(key: str, query: orm.Query, expire: int = None) -> int: return int(result) -async def db_query_cache(key: str, query: orm.Query, expire: int = None) -> list: +def db_query_cache(key: str, query: orm.Query, expire: int = None) -> list: """Store and retrieve query results via redis cache. :param key: Redis key diff --git a/aurweb/prometheus.py b/aurweb/prometheus.py index b8b7984f..d3455551 100644 --- a/aurweb/prometheus.py +++ b/aurweb/prometheus.py @@ -1,6 +1,6 @@ from typing import Any, Callable, Optional -from prometheus_client import Counter +from prometheus_client import Counter, Gauge from prometheus_fastapi_instrumentator import Instrumentator from prometheus_fastapi_instrumentator.metrics import Info from starlette.routing import Match, Route @@ -11,10 +11,26 @@ logger = aur_logging.get_logger(__name__) _instrumentator = Instrumentator() +# Custom metrics +SEARCH_REQUESTS = Counter( + "aur_search_requests", "Number of search requests by cache hit/miss", ["cache"] +) +USERS = Gauge( + "aur_users", "Number of AUR users by type", ["type"], multiprocess_mode="livemax" +) +PACKAGES = Gauge( + "aur_packages", + "Number of AUR packages by state", + ["state"], + multiprocess_mode="livemax", +) + + def instrumentator(): return _instrumentator +# FastAPI metrics # Taken from https://github.com/stephenhillier/starlette_exporter # Their license is included in LICENSES/starlette_exporter. # The code has been modified to remove child route checks diff --git a/aurweb/routers/html.py b/aurweb/routers/html.py index fc9f3519..c3bcee49 100644 --- a/aurweb/routers/html.py +++ b/aurweb/routers/html.py @@ -17,11 +17,10 @@ from sqlalchemy import case, or_ import aurweb.config import aurweb.models.package_request from aurweb import aur_logging, cookies, db, models, time, util -from aurweb.cache import db_count_cache from aurweb.exceptions import handle_form_exceptions -from aurweb.models.account_type import TRUSTED_USER_AND_DEV_ID, TRUSTED_USER_ID from aurweb.models.package_request import PENDING_ID from aurweb.packages.util import query_notified, query_voted, updated_packages +from aurweb.statistics import Statistics, update_prometheus_metrics from aurweb.templates import make_context, render_template logger = aur_logging.get_logger(__name__) @@ -87,68 +86,12 @@ async def index(request: Request): context = make_context(request, "Home") context["ssh_fingerprints"] = util.get_ssh_fingerprints() - bases = db.query(models.PackageBase) - cache_expire = aurweb.config.getint("cache", "expiry_time") + # Package statistics. - context["package_count"] = await db_count_cache( - "package_count", bases, expire=cache_expire - ) - - query = bases.filter(models.PackageBase.MaintainerUID.is_(None)) - context["orphan_count"] = await db_count_cache( - "orphan_count", query, expire=cache_expire - ) - - query = db.query(models.User) - context["user_count"] = await db_count_cache( - "user_count", query, expire=cache_expire - ) - - query = query.filter( - or_( - models.User.AccountTypeID == TRUSTED_USER_ID, - models.User.AccountTypeID == TRUSTED_USER_AND_DEV_ID, - ) - ) - context["trusted_user_count"] = await db_count_cache( - "trusted_user_count", query, expire=cache_expire - ) - - # Current timestamp. - now = time.utcnow() - - seven_days = 86400 * 7 # Seven days worth of seconds. - seven_days_ago = now - seven_days - - one_hour = 3600 - updated = bases.filter( - models.PackageBase.ModifiedTS - models.PackageBase.SubmittedTS >= one_hour - ) - - query = bases.filter(models.PackageBase.SubmittedTS >= seven_days_ago) - context["seven_days_old_added"] = await db_count_cache( - "seven_days_old_added", query, expire=cache_expire - ) - - query = updated.filter(models.PackageBase.ModifiedTS >= seven_days_ago) - context["seven_days_old_updated"] = await db_count_cache( - "seven_days_old_updated", query, expire=cache_expire - ) - - year = seven_days * 52 # Fifty two weeks worth: one year. - year_ago = now - year - query = updated.filter(models.PackageBase.ModifiedTS >= year_ago) - context["year_old_updated"] = await db_count_cache( - "year_old_updated", query, expire=cache_expire - ) - - query = bases.filter( - models.PackageBase.ModifiedTS - models.PackageBase.SubmittedTS < 3600 - ) - context["never_updated"] = await db_count_cache( - "never_updated", query, expire=cache_expire - ) + stats = Statistics(cache_expire) + for counter in stats.HOMEPAGE_COUNTERS: + context[counter] = stats.get_count(counter) # Get the 15 most recently updated packages. context["package_updates"] = updated_packages(15, cache_expire) @@ -193,7 +136,7 @@ async def index(request: Request): ) archive_time = aurweb.config.getint("options", "request_archive_time") - start = now - archive_time + start = time.utcnow() - archive_time # Package requests created by request.user. context["package_requests"] = ( @@ -269,6 +212,9 @@ async def metrics(request: Request): status_code=HTTPStatus.SERVICE_UNAVAILABLE, ) + # update prometheus gauges for packages and users + update_prometheus_metrics() + registry = CollectorRegistry() multiprocess.MultiProcessCollector(registry) data = generate_latest(registry) diff --git a/aurweb/routers/packages.py b/aurweb/routers/packages.py index 779efb4b..f1b2a138 100644 --- a/aurweb/routers/packages.py +++ b/aurweb/routers/packages.py @@ -91,9 +91,7 @@ async def packages_get( # increase the amount of time required to collect a count. # we use redis for caching the results of the query cache_expire = config.getint("cache", "expiry_time") - num_packages = await db_count_cache( - hash_query(search.query), search.query, cache_expire - ) + num_packages = db_count_cache(hash_query(search.query), search.query, cache_expire) # Apply user-specified sort column and ordering. search.sort_by(sort_by, sort_order) @@ -118,7 +116,7 @@ async def packages_get( results = results.limit(per_page).offset(offset) # we use redis for caching the results of the query - packages = await db_query_cache(hash_query(results), results, cache_expire) + packages = db_query_cache(hash_query(results), results, cache_expire) context["packages"] = packages context["packages_count"] = num_packages diff --git a/aurweb/statistics.py b/aurweb/statistics.py new file mode 100644 index 00000000..934caa37 --- /dev/null +++ b/aurweb/statistics.py @@ -0,0 +1,102 @@ +from aurweb import config, db, time +from aurweb.cache import db_count_cache +from aurweb.models import PackageBase, User +from aurweb.models.account_type import TRUSTED_USER_AND_DEV_ID, TRUSTED_USER_ID, USER_ID +from aurweb.prometheus import PACKAGES, USERS + + +class Statistics: + HOMEPAGE_COUNTERS = [ + "package_count", + "orphan_count", + "seven_days_old_added", + "seven_days_old_updated", + "year_old_updated", + "never_updated", + "user_count", + "trusted_user_count", + ] + PROMETHEUS_USER_COUNTERS = [ + ("trusted_user_count", "tu"), + ("regular_user_count", "user"), + ] + PROMETHEUS_PACKAGE_COUNTERS = [ + ("orphan_count", "orphan"), + ("never_updated", "not_updated"), + ("updated_packages", "updated"), + ] + + seven_days = 86400 * 7 + one_hour = 3600 + year = seven_days * 52 + + def __init__(self, cache_expire: int = None) -> "Statistics": + self.expiry_time = cache_expire + self.now = time.utcnow() + self.seven_days_ago = self.now - self.seven_days + self.year_ago = self.now - self.year + self.user_query = db.query(User) + self.bases_query = db.query(PackageBase) + self.updated_query = db.query(PackageBase).filter( + PackageBase.ModifiedTS - PackageBase.SubmittedTS >= self.one_hour + ) + + def get_count(self, counter: str) -> int: + query = None + match counter: + case "package_count": + query = self.bases_query + case "orphan_count": + query = self.bases_query.filter(PackageBase.MaintainerUID.is_(None)) + case "seven_days_old_added": + query = self.bases_query.filter( + PackageBase.SubmittedTS >= self.seven_days_ago + ) + case "seven_days_old_updated": + query = self.updated_query.filter( + PackageBase.ModifiedTS >= self.seven_days_ago + ) + case "year_old_updated": + query = self.updated_query.filter( + PackageBase.ModifiedTS >= self.year_ago + ) + case "never_updated": + query = self.bases_query.filter( + PackageBase.ModifiedTS - PackageBase.SubmittedTS < self.one_hour + ) + case "updated_packages": + query = self.bases_query.filter( + PackageBase.ModifiedTS - PackageBase.SubmittedTS > self.one_hour, + ~PackageBase.MaintainerUID.is_(None), + ) + case "user_count": + query = self.user_query + case "trusted_user_count": + query = self.user_query.filter( + User.AccountTypeID.in_( + ( + TRUSTED_USER_ID, + TRUSTED_USER_AND_DEV_ID, + ) + ) + ) + case "regular_user_count": + query = self.user_query.filter(User.AccountTypeID == USER_ID) + case _: + return -1 + + return db_count_cache(counter, query, expire=self.expiry_time) + + +def update_prometheus_metrics(): + cache_expire = config.getint("cache", "expiry_time") + stats = Statistics(cache_expire) + # Users gauge + for counter, utype in stats.PROMETHEUS_USER_COUNTERS: + count = stats.get_count(counter) + USERS.labels(utype).set(count) + + # Packages gauge + for counter, state in stats.PROMETHEUS_PACKAGE_COUNTERS: + count = stats.get_count(counter) + PACKAGES.labels(state).set(count) diff --git a/test/test_cache.py b/test/test_cache.py index e19fa6a2..a599ab32 100644 --- a/test/test_cache.py +++ b/test/test_cache.py @@ -31,15 +31,14 @@ def clear_fakeredis_cache(): cache._redis.flushall() -@pytest.mark.asyncio -async def test_db_count_cache(user): +def test_db_count_cache(user): query = db.query(User) # We have no cached value yet. assert cache._redis.get("key1") is None # Add to cache - assert await cache.db_count_cache("key1", query) == query.count() + assert cache.db_count_cache("key1", query) == query.count() # It's cached now. assert cache._redis.get("key1") is not None @@ -48,35 +47,34 @@ async def test_db_count_cache(user): assert cache._redis.ttl("key1") == -1 # Cache a query with an expire. - value = await cache.db_count_cache("key2", query, 100) + value = cache.db_count_cache("key2", query, 100) assert value == query.count() assert cache._redis.ttl("key2") == 100 -@pytest.mark.asyncio -async def test_db_query_cache(user): +def test_db_query_cache(user): query = db.query(User) # We have no cached value yet. assert cache._redis.get("key1") is None # Add to cache - await cache.db_query_cache("key1", query) + cache.db_query_cache("key1", query) # It's cached now. assert cache._redis.get("key1") is not None # Modify our user and make sure we got a cached value user.Username = "changed" - cached = await cache.db_query_cache("key1", query) + cached = cache.db_query_cache("key1", query) assert cached[0].Username != query.all()[0].Username # It does not expire assert cache._redis.ttl("key1") == -1 # Cache a query with an expire. - value = await cache.db_query_cache("key2", query, 100) + value = cache.db_query_cache("key2", query, 100) assert len(value) == query.count() assert value[0].Username == query.all()[0].Username @@ -90,7 +88,7 @@ async def test_db_query_cache(user): with mock.patch("aurweb.config.getint", side_effect=mock_max_search_entries): # Try to add another entry (we already have 2) - await cache.db_query_cache("key3", query) + cache.db_query_cache("key3", query) # Make sure it was not added because it exceeds our max. assert cache._redis.get("key3") is None diff --git a/test/test_metrics.py b/test/test_metrics.py index 1859d8cb..6f67d926 100644 --- a/test/test_metrics.py +++ b/test/test_metrics.py @@ -26,11 +26,10 @@ def user() -> User: yield user -@pytest.mark.asyncio -async def test_search_cache_metrics(user: User): +def test_search_cache_metrics(user: User): # Fire off 3 identical queries for caching for _ in range(3): - await db_query_cache("key", db.query(User)) + db_query_cache("key", db.query(User)) # Get metrics metrics = str(generate_latest(REGISTRY))