chore(release): prepare for 6.2.16

pygit2 and watchfiles for precompiled wheels
greenlet for python 3.13 compatibility
python-multipart for security vulnerability

Signed-off-by: Leonidas Spyropoulos <artafinde@archlinux.org>

.editorconfig

@@ -1,5 +1,5 @@
 # EditorConfig configuration for aurweb
-# https://EditorConfig.org
+# https://editorconfig.org
 
 # Top-most EditorConfig file
 root = true

.gitlab-ci.yml

@@ -19,9 +19,9 @@ variables:
 lint:
   stage: .pre
   before_script:
-    - pacman -Sy --noconfirm --noprogressbar --cachedir .pkg-cache
+    - pacman -Sy --noconfirm --noprogressbar
       archlinux-keyring
-    - pacman -Syu --noconfirm --noprogressbar --cachedir .pkg-cache
+    - pacman -Syu --noconfirm --noprogressbar
       git python python-pre-commit
   script:
     - export XDG_CACHE_HOME=.pre-commit
@@ -60,7 +60,7 @@ test:
         path: coverage.xml
 
 .init_tf: &init_tf
-  - pacman -Syu --needed --noconfirm --cachedir .pkg-cache terraform
+  - pacman -Syu --needed --noconfirm terraform
   - export TF_VAR_name="aurweb-${CI_COMMIT_REF_SLUG}"
   - TF_ADDRESS="${CI_API_V4_URL}/projects/${TF_STATE_PROJECT}/terraform/state/${CI_COMMIT_REF_SLUG}"
   - cd ci/tf
@@ -97,7 +97,7 @@ provision_review:
     - deploy_review
   script:
     - *init_tf
-    - pacman -Syu --noconfirm --needed --cachedir .pkg-cache ansible git openssh jq
+    - pacman -Syu --noconfirm --needed ansible git openssh jq
       # Get ssh key from terraform state file
     - mkdir -p ~/.ssh
     - chmod 700 ~/.ssh

aurweb/asgi.py

@@ -14,6 +14,12 @@ from fastapi import FastAPI, HTTPException, Request, Response
 from fastapi.responses import RedirectResponse
 from fastapi.staticfiles import StaticFiles
 from jinja2 import TemplateNotFound
+from opentelemetry import trace
+from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
+from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor
+from opentelemetry.sdk.resources import Resource
+from opentelemetry.sdk.trace import TracerProvider
+from opentelemetry.sdk.trace.export import BatchSpanProcessor
 from sqlalchemy import and_
 from starlette.exceptions import HTTPException as StarletteHTTPException
 from starlette.middleware.authentication import AuthenticationMiddleware
@@ -22,7 +28,6 @@ from starlette.middleware.sessions import SessionMiddleware
 import aurweb.captcha  # noqa: F401
 import aurweb.config
 import aurweb.filters  # noqa: F401
-import aurweb.pkgbase.util as pkgbaseutil
 from aurweb import aur_logging, prometheus, util
 from aurweb.aur_redis import redis_connection
 from aurweb.auth import BasicAuthBackend
@@ -54,6 +59,17 @@ instrumentator().add(prometheus.http_requests_total())
 instrumentator().instrument(app)
 
 
+# Instrument FastAPI for tracing
+FastAPIInstrumentor.instrument_app(app)
+
+resource = Resource(attributes={"service.name": "aurweb"})
+otlp_endpoint = aurweb.config.get("tracing", "otlp_endpoint")
+otlp_exporter = OTLPSpanExporter(endpoint=otlp_endpoint)
+span_processor = BatchSpanProcessor(otlp_exporter)
+trace.set_tracer_provider(TracerProvider(resource=resource))
+trace.get_tracer_provider().add_span_processor(span_processor)
+
+
 async def app_startup():
     # https://stackoverflow.com/questions/67054759/about-the-maximum-recursion-error-in-fastapi
     # Test failures have been observed by internal starlette code when
@@ -215,7 +231,13 @@ async def http_exception_handler(request: Request, exc: HTTPException) -> Respon
         if matches and len(tokens) == 2:
             try:
                 pkgbase = get_pkg_or_base(matches.group(1))
-                context = pkgbaseutil.make_context(request, pkgbase)
+                context["pkgbase"] = pkgbase
+                context["git_clone_uri_anon"] = aurweb.config.get(
+                    "options", "git_clone_uri_anon"
+                )
+                context["git_clone_uri_priv"] = aurweb.config.get(
+                    "options", "git_clone_uri_priv"
+                )
             except HTTPException:
                 pass
 

aurweb/aur_redis.py

@@ -1,4 +1,5 @@
 import fakeredis
+from opentelemetry.instrumentation.redis import RedisInstrumentor
 from redis import ConnectionPool, Redis
 
 import aurweb.config
@@ -7,6 +8,8 @@ from aurweb import aur_logging
 logger = aur_logging.get_logger(__name__)
 pool = None
 
+RedisInstrumentor().instrument()
+
 
 class FakeConnectionPool:
     """A fake ConnectionPool class which holds an internal reference

aurweb/cache.py

@@ -1,4 +1,5 @@
 import pickle
+from typing import Any, Callable
 
 from sqlalchemy import orm
 
@@ -9,6 +10,22 @@ from aurweb.prometheus import SEARCH_REQUESTS
 _redis = redis_connection()
 
 
+def lambda_cache(key: str, value: Callable[[], Any], expire: int = None) -> list:
+    """Store and retrieve lambda results via redis cache.
+
+    :param key: Redis key
+    :param value: Lambda callable returning the value
+    :param expire: Optional expiration in seconds
+    :return: result of callable or cache
+    """
+    result = _redis.get(key)
+    if result is not None:
+        return pickle.loads(result)
+
+    _redis.set(key, (pickle.dumps(result := value())), ex=expire)
+    return result
+
+
 def db_count_cache(key: str, query: orm.Query, expire: int = None) -> int:
     """Store and retrieve a query.count() via redis cache.
 

aurweb/captcha.py

@@ -3,6 +3,7 @@
 import hashlib
 
 from jinja2 import pass_context
+from sqlalchemy import func
 
 from aurweb.db import query
 from aurweb.models import User
@@ -11,7 +12,8 @@ from aurweb.templates import register_filter
 
 def get_captcha_salts():
     """Produce salts based on the current user count."""
-    count = query(User).count()
+    count = query(func.count(User.ID)).scalar()
+
     salts = []
     for i in range(0, 6):
         salts.append(f"aurweb-{count - i}")

aurweb/db.py

@@ -298,9 +298,12 @@ def get_engine(dbname: str = None, echo: bool = False):
             connect_args["check_same_thread"] = False
 
         kwargs = {"echo": echo, "connect_args": connect_args}
+        from opentelemetry.instrumentation.sqlalchemy import SQLAlchemyInstrumentor
         from sqlalchemy import create_engine
 
-        _engines[dbname] = create_engine(get_sqlalchemy_url(), **kwargs)
+        engine = create_engine(get_sqlalchemy_url(), **kwargs)
+        SQLAlchemyInstrumentor().instrument(engine=engine)
+        _engines[dbname] = engine
 
         if is_sqlite:  # pragma: no cover
             setup_sqlite(_engines.get(dbname))

aurweb/pkgbase/util.py

@@ -2,6 +2,7 @@ from typing import Any
 
 from fastapi import Request
 from sqlalchemy import and_
+from sqlalchemy.orm import joinedload
 
 from aurweb import config, db, defaults, l10n, time, util
 from aurweb.models import PackageBase, User
@@ -26,6 +27,8 @@ def make_context(
     if not context:
         context = _make_context(request, pkgbase.Name)
 
+    is_authenticated = request.user.is_authenticated()
+
     # Per page and offset.
     offset, per_page = util.sanitize_params(
         request.query_params.get("O", defaults.O),
@@ -38,12 +41,15 @@ def make_context(
     context["pkgbase"] = pkgbase
     context["comaintainers"] = [
         c.User
-        for c in pkgbase.comaintainers.order_by(
+        for c in pkgbase.comaintainers.options(joinedload(PackageComaintainer.User))
-            PackageComaintainer.Priority.asc()
+        .order_by(PackageComaintainer.Priority.asc())
-        ).all()
+        .all()
     ]
-    context["unflaggers"] = context["comaintainers"].copy()
+    if is_authenticated:
-    context["unflaggers"].extend([pkgbase.Maintainer, pkgbase.Flagger])
+        context["unflaggers"] = context["comaintainers"].copy()
+        context["unflaggers"].extend([pkgbase.Maintainer, pkgbase.Flagger])
+    else:
+        context["unflaggers"] = []
 
     context["packages_count"] = pkgbase.packages.count()
     context["keywords"] = pkgbase.keywords
@@ -60,17 +66,28 @@ def make_context(
     ).order_by(PackageComment.CommentTS.desc())
 
     context["is_maintainer"] = bool(request.user == pkgbase.Maintainer)
-    context["notified"] = request.user.notified(pkgbase)
+    if is_authenticated:
+        context["notified"] = request.user.notified(pkgbase)
+    else:
+        context["notified"] = False
 
     context["out_of_date"] = bool(pkgbase.OutOfDateTS)
 
-    context["voted"] = request.user.package_votes.filter(
+    if is_authenticated:
-        PackageVote.PackageBaseID == pkgbase.ID
+        context["voted"] = db.query(
-    ).scalar()
+            request.user.package_votes.filter(
+                PackageVote.PackageBaseID == pkgbase.ID
+            ).exists()
+        ).scalar()
+    else:
+        context["voted"] = False
 
-    context["requests"] = pkgbase.requests.filter(
+    if is_authenticated:
-        and_(PackageRequest.Status == PENDING_ID, PackageRequest.ClosedTS.is_(None))
+        context["requests"] = pkgbase.requests.filter(
-    ).count()
+            and_(PackageRequest.Status == PENDING_ID, PackageRequest.ClosedTS.is_(None))
+        ).count()
+    else:
+        context["requests"] = []
 
     context["popularity"] = popularity(pkgbase, time.utcnow())
 

aurweb/routers/packages.py

@@ -190,6 +190,17 @@ async def package(
     if not all_deps:
         deps = deps.limit(max_listing)
     context["dependencies"] = deps.all()
+    # Existing dependencies to avoid multiple lookups
+    context["dependencies_names_from_aur"] = [
+        item.Name
+        for item in db.query(models.Package)
+        .filter(
+            models.Package.Name.in_(
+                pkg.package_dependencies.with_entities(models.PackageDependency.DepName)
+            )
+        )
+        .all()
+    ]
 
     # Package requirements (other packages depend on this one).
     reqs = pkgutil.pkg_required(pkg.Name, [p.RelName for p in rels_data.get("p", [])])

aurweb/routers/rss.py

@@ -2,7 +2,8 @@ from fastapi import APIRouter, Request
 from fastapi.responses import Response
 from feedgen.feed import FeedGenerator
 
-from aurweb import db, filters
+from aurweb import config, db, filters
+from aurweb.cache import lambda_cache
 from aurweb.models import Package, PackageBase
 
 router = APIRouter()
@@ -56,9 +57,11 @@ async def rss(request: Request):
         )
     )
 
-    feed = make_rss_feed(request, packages)
+    # we use redis for caching the results of the feedgen
-    response = Response(feed, media_type="application/rss+xml")
+    cache_expire = config.getint("cache", "expiry_time_rss", 300)
+    feed = lambda_cache("rss", lambda: make_rss_feed(request, packages), cache_expire)
 
+    response = Response(feed, media_type="application/rss+xml")
     return response
 
 
@@ -76,7 +79,11 @@ async def rss_modified(request: Request):
         )
     )
 
-    feed = make_rss_feed(request, packages)
+    # we use redis for caching the results of the feedgen
-    response = Response(feed, media_type="application/rss+xml")
+    cache_expire = config.getint("cache", "expiry_time_rss", 300)
+    feed = lambda_cache(
+        "rss_modified", lambda: make_rss_feed(request, packages), cache_expire
+    )
 
+    response = Response(feed, media_type="application/rss+xml")
     return response

aurweb/schema.py

@@ -183,6 +183,8 @@ PackageBases = Table(
     Index("BasesNumVotes", "NumVotes"),
     Index("BasesPackagerUID", "PackagerUID"),
     Index("BasesSubmitterUID", "SubmitterUID"),
+    Index("BasesSubmittedTS", "SubmittedTS"),
+    Index("BasesModifiedTS", "ModifiedTS"),
     mysql_engine="InnoDB",
     mysql_charset="utf8mb4",
     mysql_collate="utf8mb4_general_ci",

aurweb/spawn.py

@@ -51,46 +51,46 @@ def generate_nginx_config():
     fastapi_bind = aurweb.config.get("fastapi", "bind_address")
     fastapi_host = fastapi_bind.split(":")[0]
     config_path = os.path.join(temporary_dir, "nginx.conf")
-    config = open(config_path, "w")
+    with open(config_path, "w") as config:
-    # We double nginx's braces because they conflict with Python's f-strings.
+        # We double nginx's braces because they conflict with Python's f-strings.
-    config.write(
+        config.write(
-        f"""
+            f"""
-        events {{}}
+            events {{}}
-        daemon off;
+            daemon off;
-        error_log /dev/stderr info;
+            error_log /dev/stderr info;
-        pid {os.path.join(temporary_dir, "nginx.pid")};
+            pid {os.path.join(temporary_dir, "nginx.pid")};
-        http {{
+            http {{
-            access_log /dev/stdout;
+                access_log /dev/stdout;
-            client_body_temp_path {os.path.join(temporary_dir, "client_body")};
+                client_body_temp_path {os.path.join(temporary_dir, "client_body")};
-            proxy_temp_path {os.path.join(temporary_dir, "proxy")};
+                proxy_temp_path {os.path.join(temporary_dir, "proxy")};
-            fastcgi_temp_path {os.path.join(temporary_dir, "fastcgi")}1 2;
+                fastcgi_temp_path {os.path.join(temporary_dir, "fastcgi")}1 2;
-            uwsgi_temp_path {os.path.join(temporary_dir, "uwsgi")};
+                uwsgi_temp_path {os.path.join(temporary_dir, "uwsgi")};
-            scgi_temp_path {os.path.join(temporary_dir, "scgi")};
+                scgi_temp_path {os.path.join(temporary_dir, "scgi")};
-            server {{
+                server {{
-                listen {fastapi_host}:{FASTAPI_NGINX_PORT};
+                    listen {fastapi_host}:{FASTAPI_NGINX_PORT};
-                location / {{
+                    location / {{
-                    try_files $uri @proxy_to_app;
+                        try_files $uri @proxy_to_app;
-                }}
+                    }}
-                location @proxy_to_app {{
+                    location @proxy_to_app {{
-                    proxy_set_header Host $http_host;
+                        proxy_set_header Host $http_host;
-                    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+                        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
-                    proxy_set_header X-Forwarded-Proto $scheme;
+                        proxy_set_header X-Forwarded-Proto $scheme;
-                    proxy_redirect off;
+                        proxy_redirect off;
-                    proxy_buffering off;
+                        proxy_buffering off;
-                    proxy_pass http://{fastapi_bind};
+                        proxy_pass http://{fastapi_bind};
+                    }}
                 }}
             }}
-        }}
+        """
-    """
+        )
-    )
     return config_path
 
 
-def spawn_child(args):
+def spawn_child(_args):
     """Open a subprocess and add it to the global state."""
     if verbosity >= 1:
-        print(f":: Spawning {args}", file=sys.stderr)
+        print(f":: Spawning {_args}", file=sys.stderr)
-    children.append(subprocess.Popen(args))
+    children.append(subprocess.Popen(_args))
 
 
 def start():
@@ -171,17 +171,17 @@ def start():
     )
 
 
-def _kill_children(
+def _kill_children(_children: Iterable, exceptions=None) -> list[Exception]:
-    children: Iterable, exceptions: list[Exception] = []
-) -> list[Exception]:
     """
     Kill each process found in `children`.
 
-    :param children: Iterable of child processes
+    :param _children: Iterable of child processes
     :param exceptions: Exception memo
     :return: `exceptions`
     """
-    for p in children:
+    if exceptions is None:
+        exceptions = []
+    for p in _children:
         try:
             p.terminate()
             if verbosity >= 1:
@@ -191,17 +191,17 @@ def _kill_children(
     return exceptions
 
 
-def _wait_for_children(
+def _wait_for_children(_children: Iterable, exceptions=None) -> list[Exception]:
-    children: Iterable, exceptions: list[Exception] = []
-) -> list[Exception]:
     """
     Wait for each process to end found in `children`.
 
-    :param children: Iterable of child processes
+    :param _children: Iterable of child processes
     :param exceptions: Exception memo
     :return: `exceptions`
     """
-    for p in children:
+    if exceptions is None:
+        exceptions = []
+    for p in _children:
         try:
             rc = p.wait()
             if rc != 0 and rc != -15:

conf/config.defaults

@@ -175,3 +175,8 @@ max_search_entries = 50000
 expiry_time_search = 600
 ; number of seconds after a cache entry for statistics queries expires, default is 5 minutes
 expiry_time_statistics = 300
+; number of seconds after a cache entry for rss queries expires, default is 5 minutes
+expiry_time_rss = 300
+
+[tracing]
+otlp_endpoint = http://localhost:4318/v1/traces

conf/config.dev

@@ -73,3 +73,6 @@ pkgnames-repo = pkgnames.git
 
 [aurblup]
 db-path = YOUR_AUR_ROOT/aurblup/
+
+[tracing]
+otlp_endpoint = http://tempo:4318/v1/traces

docker-compose.aur-dev.yml

@@ -1,5 +1,4 @@
-version: "3.8"
+---
-
 services:
   ca:
     volumes:

docker-compose.override.yml

@@ -1,16 +1,10 @@
-version: "3.8"
+---
-
 services:
   ca:
     volumes:
       - ./data:/data
       - step:/root/.step
 
-  mariadb_init:
-    depends_on:
-      mariadb:
-        condition: service_healthy
-
   git:
     volumes:
       - git_data:/aurweb/aur.git
@@ -21,9 +15,6 @@ services:
       - git_data:/aurweb/aur.git
       - ./data:/data
       - smartgit_run:/var/run/smartgit
-    depends_on:
-      mariadb:
-        condition: service_healthy
 
   fastapi:
     volumes:

docker-compose.yml

@@ -1,3 +1,4 @@
+---
 #
 # Docker service definitions for the aurweb project.
 #
@@ -16,8 +17,6 @@
 #
 # Copyright (C) 2021 aurweb Development
 # All Rights Reserved.
-version: "3.8"
-
 services:
   aurweb-image:
     build: .
@@ -49,7 +48,7 @@ services:
     image: aurweb:latest
     init: true
     entrypoint: /docker/mariadb-entrypoint.sh
-    command: /usr/bin/mysqld_safe --datadir=/var/lib/mysql
+    command: /usr/bin/mariadbd-safe --datadir=/var/lib/mysql
     ports:
       # This will expose mariadbd on 127.0.0.1:13306 in the host.
       # Ex: `mysql -uaur -paur -h 127.0.0.1 -P 13306 aurweb`
@@ -81,7 +80,7 @@ services:
     environment:
       - MARIADB_PRIVILEGED=1
     entrypoint: /docker/mariadb-entrypoint.sh
-    command: /usr/bin/mysqld_safe --datadir=/var/lib/mysql
+    command: /usr/bin/mariadbd-safe --datadir=/var/lib/mysql
     ports:
       # This will expose mariadbd on 127.0.0.1:13307 in the host.
       # Ex: `mysql -uaur -paur -h 127.0.0.1 -P 13306 aurweb`
@@ -107,8 +106,10 @@ services:
       test: "bash /docker/health/sshd.sh"
       interval: 3s
     depends_on:
+      mariadb:
+        condition: service_healthy
       mariadb_init:
-        condition: service_started
+        condition: service_completed_successfully
     volumes:
       - mariadb_run:/var/run/mysqld
 
@@ -122,6 +123,9 @@ services:
     healthcheck:
       test: "bash /docker/health/smartgit.sh"
       interval: 3s
+    depends_on:
+      mariadb:
+        condition: service_healthy
 
   cgit-fastapi:
     image: aurweb:latest
@@ -152,8 +156,10 @@ services:
     entrypoint: /docker/cron-entrypoint.sh
     command: /docker/scripts/run-cron.sh
     depends_on:
+      mariadb:
+        condition: service_healthy
       mariadb_init:
-        condition: service_started
+        condition: service_completed_successfully
     volumes:
       - ./aurweb:/aurweb/aurweb
       - mariadb_run:/var/run/mysqld
@@ -182,6 +188,12 @@ services:
         condition: service_healthy
       cron:
         condition: service_started
+      mariadb:
+        condition: service_healthy
+      mariadb_init:
+        condition: service_completed_successfully
+      tempo:
+        condition: service_healthy
     volumes:
       - archives:/var/lib/aurweb/archives
       - mariadb_run:/var/run/mysqld
@@ -281,6 +293,56 @@ services:
       - ./test:/aurweb/test
       - ./templates:/aurweb/templates
 
+  grafana:
+    # TODO: check if we need init: true
+    image: grafana/grafana:11.1.3
+    environment:
+      - GF_AUTH_ANONYMOUS_ENABLED=true
+      - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin
+      - GF_AUTH_DISABLE_LOGIN_FORM=true
+      - GF_LOG_LEVEL=warn
+    # check if depends ar ecorrect, does stopping or restarting a child exit grafana?
+    depends_on:
+      prometheus:
+        condition: service_healthy
+      tempo:
+        condition: service_healthy
+    ports:
+      - "127.0.0.1:3000:3000"
+    volumes:
+      - ./docker/config/grafana/datasources:/etc/grafana/provisioning/datasources
+
+  prometheus:
+    image: prom/prometheus:latest
+    command:
+      - --config.file=/etc/prometheus/prometheus.yml
+      - --web.enable-remote-write-receiver
+      - --web.listen-address=prometheus:9090
+    healthcheck:
+      # TODO: check if there is a status route
+      test: "sh /docker/health/prometheus.sh"
+      interval: 3s
+    ports:
+      - "127.0.0.1:9090:9090"
+    volumes:
+      - ./docker/config/prometheus.yml:/etc/prometheus/prometheus.yml
+      - ./docker/health/prometheus.sh:/docker/health/prometheus.sh
+
+  tempo:
+    image: grafana/tempo:2.5.0
+    command:
+      - -config.file=/etc/tempo/config.yml
+    healthcheck:
+      # TODO: check if there is a status route
+      test: "sh /docker/health/tempo.sh"
+      interval: 3s
+    ports:
+      - "127.0.0.1:3200:3200"
+      - "127.0.0.1:4318:4318"
+    volumes:
+      - ./docker/config/tempo.yml:/etc/tempo/config.yml
+      - ./docker/health/tempo.sh:/docker/health/tempo.sh
+
 volumes:
   mariadb_test_run: {}
   mariadb_run: {} # Share /var/run/mysqld/mysqld.sock

docker/README.md

@@ -47,7 +47,7 @@ Luckily such data can be generated.
 docker compose exec fastapi /bin/bash
 pacman -S words fortune-mod
 ./schema/gendummydata.py dummy.sql
-mysql aurweb < dummy.sql
+mariadb aurweb < dummy.sql
 ```
 
 The generation script may prompt you to install other Arch packages before it

docker/config/grafana/datasources/datasource.yml

@@ -0,0 +1,42 @@
+---
+apiVersion: 1
+
+deleteDatasources:
+  - name: Prometheus
+  - name: Tempo
+
+datasources:
+  - name: Prometheus
+    type: prometheus
+    uid: prometheus
+    access: proxy
+    url: http://prometheus:9090
+    orgId: 1
+    editable: false
+    jsonData:
+      timeInterval: 1m
+  - name: Tempo
+    type: tempo
+    uid: tempo
+    access: proxy
+    url: http://tempo:3200
+    orgId: 1
+    editable: false
+    jsonData:
+      tracesToMetrics:
+        datasourceUid: 'prometheus'
+        spanStartTimeShift: '1h'
+        spanEndTimeShift: '-1h'
+      serviceMap:
+        datasourceUid: 'prometheus'
+      nodeGraph:
+        enabled: true
+      search:
+        hide: false
+      traceQuery:
+        timeShiftEnabled: true
+        spanStartTimeShift: '1h'
+        spanEndTimeShift: '-1h'
+      spanBar:
+        type: 'Tag'
+        tag: 'http.path'

docker/config/prometheus.yml

@@ -0,0 +1,15 @@
+---
+global:
+  scrape_interval: 60s
+
+scrape_configs:
+  - job_name: tempo
+    static_configs:
+      - targets: ['tempo:3200']
+        labels:
+          instance: tempo
+  - job_name: aurweb
+    static_configs:
+      - targets: ['fastapi:8000']
+        labels:
+          instance: aurweb

docker/config/tempo.yml

@@ -0,0 +1,54 @@
+---
+stream_over_http_enabled: true
+server:
+  http_listen_address: tempo
+  http_listen_port: 3200
+  log_level: info
+
+query_frontend:
+  search:
+    duration_slo: 5s
+    throughput_bytes_slo: 1.073741824e+09
+  trace_by_id:
+    duration_slo: 5s
+
+distributor:
+  receivers:
+    otlp:
+      protocols:
+        http:
+          endpoint: tempo:4318
+  log_received_spans:
+    enabled: false
+  metric_received_spans:
+    enabled: false
+
+ingester:
+  max_block_duration: 5m
+
+compactor:
+  compaction:
+    block_retention: 1h
+
+metrics_generator:
+  registry:
+    external_labels:
+      source: tempo
+  storage:
+    path: /tmp/tempo/generator/wal
+    remote_write:
+      - url: http://prometheus:9090/api/v1/write
+        send_exemplars: true
+  traces_storage:
+    path: /tmp/tempo/generator/traces
+
+storage:
+  trace:
+    backend: local
+    wal:
+      path: /tmp/tempo/wal
+    local:
+      path: /tmp/tempo/blocks
+
+overrides:
+  metrics_generator_processors: [service-graphs, span-metrics, local-blocks]

docker/health/mariadb.sh

@@ -1,2 +1,2 @@
 #!/bin/bash
-exec mysqladmin ping --silent
+exec mariadb-admin ping --silent

docker/health/prometheus.sh

@@ -0,0 +1,2 @@
+#!/bin/sh
+exec wget -q http://prometheus:9090/status -O /dev/null

docker/health/tempo.sh

@@ -0,0 +1,2 @@
+#!/bin/sh
+exec wget -q http://tempo:3200/status -O /dev/null

docker/mariadb-entrypoint.sh

@@ -6,8 +6,8 @@ MYSQL_DATA=/var/lib/mysql
 mariadb-install-db --user=mysql --basedir=/usr --datadir=$MYSQL_DATA
 
 # Start it up.
-mysqld_safe --datadir=$MYSQL_DATA --skip-networking &
+mariadbd-safe --datadir=$MYSQL_DATA --skip-networking &
-while ! mysqladmin ping 2>/dev/null; do
+while ! mariadb-admin ping 2>/dev/null; do
     sleep 1s
 done
 
@@ -15,17 +15,17 @@ done
 DATABASE="aurweb" # Persistent database for fastapi.
 
 echo "Taking care of primary database '${DATABASE}'..."
-mysql -u root -e "CREATE USER IF NOT EXISTS 'aur'@'localhost' IDENTIFIED BY 'aur';"
+mariadb -u root -e "CREATE USER IF NOT EXISTS 'aur'@'localhost' IDENTIFIED BY 'aur';"
-mysql -u root -e "CREATE USER IF NOT EXISTS 'aur'@'%' IDENTIFIED BY 'aur';"
+mariadb -u root -e "CREATE USER IF NOT EXISTS 'aur'@'%' IDENTIFIED BY 'aur';"
-mysql -u root -e "CREATE DATABASE IF NOT EXISTS $DATABASE;"
+mariadb -u root -e "CREATE DATABASE IF NOT EXISTS $DATABASE;"
 
-mysql -u root -e "CREATE USER IF NOT EXISTS 'aur'@'%' IDENTIFIED BY 'aur';"
+mariadb -u root -e "CREATE USER IF NOT EXISTS 'aur'@'%' IDENTIFIED BY 'aur';"
-mysql -u root -e "GRANT ALL ON aurweb.* TO 'aur'@'localhost';"
+mariadb -u root -e "GRANT ALL ON aurweb.* TO 'aur'@'localhost';"
-mysql -u root -e "GRANT ALL ON aurweb.* TO 'aur'@'%';"
+mariadb -u root -e "GRANT ALL ON aurweb.* TO 'aur'@'%';"
 
-mysql -u root -e "CREATE USER IF NOT EXISTS 'root'@'%' IDENTIFIED BY 'aur';"
+mariadb -u root -e "CREATE USER IF NOT EXISTS 'root'@'%' IDENTIFIED BY 'aur';"
-mysql -u root -e "GRANT ALL ON *.* TO 'root'@'%' WITH GRANT OPTION;"
+mariadb -u root -e "GRANT ALL ON *.* TO 'root'@'%' WITH GRANT OPTION;"
 
-mysqladmin -uroot shutdown
+mariadb-admin -uroot shutdown
 
 exec "$@"

docker/scripts/install-deps.sh

@@ -13,7 +13,7 @@ pacman -Sy --noconfirm --noprogressbar archlinux-keyring
 
 # Install other OS dependencies.
 pacman -Syu --noconfirm --noprogressbar \
-    --cachedir .pkg-cache git gpgme nginx redis openssh \
+    git gpgme nginx redis openssh \
     mariadb mariadb-libs cgit-aurweb uwsgi uwsgi-plugin-cgi \
     python-pip pyalpm python-srcinfo curl libeatmydata cronie \
     python-poetry python-poetry-core step-cli step-ca asciidoc \

migrations/versions/38e5b9982eea_add_indicies_on_packagebases_for_rss_.py

@@ -0,0 +1,29 @@
+"""add indices on PackageBases for RSS order by
+
+Revision ID: 38e5b9982eea
+Revises: 7d65d35fae45
+Create Date: 2024-08-03 01:35:39.104283
+
+"""
+
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "38e5b9982eea"
+down_revision = "7d65d35fae45"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_index("BasesModifiedTS", "PackageBases", ["ModifiedTS"], unique=False)
+    op.create_index("BasesSubmittedTS", "PackageBases", ["SubmittedTS"], unique=False)
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_index("BasesSubmittedTS", table_name="PackageBases")
+    op.drop_index("BasesModifiedTS", table_name="PackageBases")
+    # ### end Alembic commands ###

pyproject.toml

@@ -16,7 +16,7 @@ combine_as_imports = true
 #
 [tool.poetry]
 name = "aurweb"
-version = "v6.2.11"
+version = "v6.2.16"
 license = "GPL-2.0-only"
 description = "Source code for the Arch User Repository's website"
 homepage = "https://aur.archlinux.org"
@@ -52,13 +52,13 @@ build-backend = "poetry.masonry.api"
 "Request Mailing List" = "https://lists.archlinux.org/listinfo/aur-requests"
 
 [tool.poetry.dependencies]
-python = ">=3.9,<3.13"
+python = ">=3.10,<3.14"
 
 # poetry-dynamic-versioning is used to produce tool.poetry.version
 # based on git tags.
 
 # General
-aiofiles = "^23.2.1"
+aiofiles = "^24.0.0"
 asgiref = "^3.8.1"
 bcrypt = "^4.1.2"
 bleach = "^6.1.0"
@@ -69,34 +69,42 @@ httpx = "^0.27.0"
 itsdangerous = "^2.1.2"
 lxml = "^5.2.1"
 orjson = "^3.10.0"
-protobuf = "^5.26.1"
+pygit2 = "^1.17.0"
-pygit2 = "^1.14.1"
+python-multipart = "0.0.19"
-python-multipart = "^0.0.9"
 redis = "^5.0.3"
 requests = "^2.31.0"
 paginate = "^0.5.6"
 
 # SQL
 alembic = "^1.13.1"
-mysqlclient = "^2.2.4"
+mysqlclient = "^2.2.3"
 Authlib = "^1.3.0"
 Jinja2 = "^3.1.3"
 Markdown = "^3.6"
 Werkzeug = "^3.0.2"
 SQLAlchemy = "^1.4.52"
+greenlet = "3.1.1" # Explicitly add greenlet (dependency of SQLAlchemy) for python 3.13 support
 
 # ASGI
-uvicorn = "^0.29.0"
+uvicorn = "^0.30.0"
-gunicorn = "^21.2.0"
+gunicorn = "^22.0.0"
-Hypercorn = "^0.16.0"
+Hypercorn = "^0.17.0"
-prometheus-fastapi-instrumentator = "^7.0.0"
 pytest-xdist = "^3.5.0"
 filelock = "^3.13.3"
 posix-ipc = "^1.1.1"
 pyalpm = "^0.10.6"
-fastapi = "^0.110.0"
+fastapi = "^0.112.0"
 srcinfo = "^0.1.2"
-tomlkit = "^0.12.0"
+tomlkit = "^0.13.0"
+
+# Tracing
+prometheus-fastapi-instrumentator = "^7.0.0"
+opentelemetry-api = "^1.26.0"
+opentelemetry-sdk = "^1.26.0"
+opentelemetry-exporter-otlp-proto-http = "^1.26.0"
+opentelemetry-instrumentation-fastapi = "^0.47b0"
+opentelemetry-instrumentation-redis = "^0.47b0"
+opentelemetry-instrumentation-sqlalchemy = "^0.47b0"
 
 [tool.poetry.dev-dependencies]
 coverage = "^7.4.4"
@@ -104,7 +112,7 @@ pytest = "^8.1.1"
 pytest-asyncio = "^0.23.0"
 pytest-cov = "^5.0.0"
 pytest-tap = "^3.4"
-watchfiles = "^0.21.0"
+watchfiles = "^1.0.4"
 
 [tool.poetry.scripts]
 aurweb-git-auth = "aurweb.git.auth:main"

templates/partials/packages/actions.html

@@ -24,71 +24,73 @@
                     {{ "Search wiki" | tr }}
                 </a>
             </li>
-            {% if not out_of_date %}
+            {% if request.user.is_authenticated() %}
+				{% if not out_of_date %}
+					<li>
+						<a href="/pkgbase/{{ pkgbase.Name }}/flag">
+							{{ "Flag package out-of-date" | tr }}
+						</a>
+					</li>
+				{% else %}
+					<li>
+						<span class="flagged">
+							<a href="/pkgbase/{{ pkgbase.Name }}/flag-comment">
+								{{
+								"Flagged out-of-date (%s)"
+								| tr | format(date_strftime(pkgbase.OutOfDateTS, "%Y-%m-%d"))
+								}}
+							</a>
+						</span>
+					</li>
+					{% if request.user.has_credential(creds.PKGBASE_UNFLAG, approved=unflaggers) %}
+					<li>
+						<form action="/pkgbase/{{ pkgbase.Name }}/unflag" method="post">
+							<input class="button text-button"
+								   type="submit"
+								   name="do_UnFlag"
+								   value="{{ 'Unflag package' | tr }}"
+							/>
+						</form>
+					</li>
+					{% endif %}
+				{% endif %}
                 <li>
-                    <a href="/pkgbase/{{ pkgbase.Name }}/flag">
+                    {% if not voted %}
-                        {{ "Flag package out-of-date" | tr }}
+                    <form action="/pkgbase/{{ pkgbase.Name }}/vote" method="post">
-                    </a>
+                        <input type="submit"
-                </li>
+                               class="button text-button"
-            {% else %}
+                               name="do_Vote"
-                <li>
+                               value="{{ 'Vote for this package' | tr }}" />
-                    <span class="flagged">
-                        <a href="/pkgbase/{{ pkgbase.Name }}/flag-comment">
-                            {{
-                            "Flagged out-of-date (%s)"
-                            | tr | format(date_strftime(pkgbase.OutOfDateTS, "%Y-%m-%d"))
-                            }}
-                        </a>
-                    </span>
-                </li>
-                {% if request.user.has_credential(creds.PKGBASE_UNFLAG, approved=unflaggers) %}
-                <li>
-                    <form action="/pkgbase/{{ pkgbase.Name }}/unflag" method="post">
-                        <input class="button text-button"
-                               type="submit"
-                               name="do_UnFlag"
-                               value="{{ 'Unflag package' | tr }}"
-                        />
                     </form>
+                    {% else %}
+                    <form action="/pkgbase/{{ pkgbase.Name }}/unvote" method="post">
+                       <input type="submit"
+                              class="button text-button"
+                              name="do_UnVote"
+                              value="{{ 'Remove vote' | tr }}" />
+                    </form>
+                    {% endif %}
+                </li>
+                <li>
+                    {% if notified %}
+                        <form action="/pkgbase/{{ pkgbase.Name }}/unnotify" method="post">
+                            <input type="submit"
+                                   class="button text-button"
+                                   name="do_UnNotify"
+                                   value="{{ 'Disable notifications' | tr }}"
+                            />
+                        </form>
+                    {% else %}
+                        <form action="/pkgbase/{{ pkgbase.Name }}/notify" method="post">
+                            <input type="submit"
+                                   class="button text-button"
+                                   name="do_Notify"
+                                   value="{{ 'Enable notifications' | tr }}"
+                            />
+                        </form>
+                    {% endif %}
                 </li>
-                {% endif %}
             {% endif %}
-            <li>
-                {% if not voted %}
-                <form action="/pkgbase/{{ pkgbase.Name }}/vote" method="post">
-                    <input type="submit"
-                           class="button text-button"
-                           name="do_Vote"
-                           value="{{ 'Vote for this package' | tr }}" />
-                </form>
-                {% else %}
-                <form action="/pkgbase/{{ pkgbase.Name }}/unvote" method="post">
-                    <input type="submit"
-                           class="button text-button"
-                           name="do_UnVote"
-                           value="{{ 'Remove vote' | tr }}" />
-                </form>
-                {% endif %}
-            </li>
-            <li>
-                {% if notified %}
-                    <form action="/pkgbase/{{ pkgbase.Name }}/unnotify" method="post">
-                        <input type="submit"
-                               class="button text-button"
-                               name="do_UnNotify"
-                               value="{{ 'Disable notifications' | tr }}"
-                        />
-                    </form>
-                {% else %}
-                    <form action="/pkgbase/{{ pkgbase.Name }}/notify" method="post">
-                        <input type="submit"
-                               class="button text-button"
-                               name="do_Notify"
-                               value="{{ 'Enable notifications' | tr }}"
-                        />
-                    </form>
-                {% endif %}
-            </li>
             {% if request.user.has_credential(creds.PKGBASE_EDIT_COMAINTAINERS, approved=[pkgbase.Maintainer]) %}
                 <li>
                     <a href="/pkgbase/{{ pkgbase.Name }}/comaintainers">
@@ -111,11 +113,13 @@
                     </li>
                 {% endif %}
             {% endif %}
-            <li>
+            {% if request.user.is_authenticated() %}
-                <a href="/pkgbase/{{ pkgbase.Name }}/request?{{ {'next': '/pkgbase/%s' | format(pkgbase.Name)} | urlencode }}">
+                <li>
-                    {{ "Submit Request" | tr }}
+                    <a href="/pkgbase/{{ pkgbase.Name }}/request?{{ {'next': '/pkgbase/%s' | format(pkgbase.Name)} | urlencode }}">
-                </a>
+                        {{ "Submit Request" | tr }}
-            </li>
+                    </a>
+                </li>
+            {% endif %}
             {% if request.user.has_credential(creds.PKGBASE_DELETE) %}
                 <li>
                     <a href="/pkgbase/{{ pkgbase.Name }}/delete?next=/packages">

templates/partials/packages/package_metadata.html

@@ -20,7 +20,7 @@
                     {% endif %}
                 {% else -%}
                     </a>
-                    {%- if dep.is_aur_package() -%}
+                    {%- if dep.DepName in dependencies_names_from_aur -%}
                         <sup><small>AUR</small></sup>
                     {% endif %}
                 {% endif %}

templates/requests.html

@@ -119,12 +119,15 @@
                             <td>
                                 {# Filed by #}
                                 {# If the record has an associated User, display a link to that user. #}
-                                {# Otherwise, display nothing (an empty column). #}
+                                {# Otherwise, display "(deleted)". #}
                                 {% if result.User %}
                                     <a href="/account/{{ result.User.Username }}">
                                         {{ result.User.Username }}
-                                    </a>&nbsp;
+                                    </a>
+                                {% else %}
+                                    <i>(deleted)</i>
                                 {% endif %}
+                                &nbsp;
                                 <a target="_blank" rel="noopener noreferrer" href="{{ result.ml_message_url() }}">
                                     (PRQ#{{ result.ID }})
                                 </a>

test/test_packages_routes.py

@@ -507,7 +507,9 @@ def test_package_requests_display(
     client: TestClient, user: User, package: Package, pkgreq: PackageRequest
 ):
     # Test that a single request displays "1 pending request".
+    cookies = {"AURSID": user.login(Request(), "testPassword")}
     with client as request:
+        request.cookies = cookies
         resp = request.get(package_endpoint(package))
     assert resp.status_code == int(HTTPStatus.OK)
 
@@ -530,6 +532,7 @@ def test_package_requests_display(
 
     # Test that a two requests display "2 pending requests".
     with client as request:
+        request.cookies = cookies
         resp = request.get(package_endpoint(package))
     assert resp.status_code == int(HTTPStatus.OK)
 

test/test_requests.py

@@ -834,6 +834,16 @@ def test_requests(
     rows = root.xpath('//table[@class="results"]/tbody/tr')
     assert len(rows) == 5  # There are five records left on the second page.
 
+    # Delete requesters user account and check output
+    with db.begin():
+        db.delete(requests[0].User)
+
+    with client as request:
+        request.cookies = cookies
+        resp = request.get("/requests")
+
+    assert "(deleted)" in resp.text
+
 
 def test_requests_with_filters(
     client: TestClient,