From 3aa8d523f5ee6beb13a9e981e2f8754f79c54578 Mon Sep 17 00:00:00 2001 From: Colin Woodbury Date: Mon, 21 Feb 2022 16:49:38 -0800 Subject: [PATCH 1/5] change(rpc): `search` module reformatting --- aurweb/packages/search.py | 93 +++++++++++++++++++++------------------ 1 file changed, 50 insertions(+), 43 deletions(-) diff --git a/aurweb/packages/search.py b/aurweb/packages/search.py index 4a6eb75f..5ba72652 100644 --- a/aurweb/packages/search.py +++ b/aurweb/packages/search.py @@ -12,7 +12,7 @@ from aurweb.models.package_vote import PackageVote class PackageSearch: - """ A Package search query builder. """ + """A Package search query builder.""" # A constant mapping of short to full name sort orderings. FULL_SORT_ORDER = {"d": "desc", "a": "asc"} @@ -24,14 +24,18 @@ class PackageSearch: if self.user: self.query = self.query.join( PackageVote, - and_(PackageVote.PackageBaseID == PackageBase.ID, - PackageVote.UsersID == self.user.ID), - isouter=True + and_( + PackageVote.PackageBaseID == PackageBase.ID, + PackageVote.UsersID == self.user.ID, + ), + isouter=True, ).join( PackageNotification, - and_(PackageNotification.PackageBaseID == PackageBase.ID, - PackageNotification.UserID == self.user.ID), - isouter=True + and_( + PackageNotification.PackageBaseID == PackageBase.ID, + PackageNotification.UserID == self.user.ID, + ), + isouter=True, ) self.ordering = "d" @@ -47,7 +51,7 @@ class PackageSearch: "m": self._search_by_maintainer, "c": self._search_by_comaintainer, "M": self._search_by_co_or_maintainer, - "s": self._search_by_submitter + "s": self._search_by_submitter, } # Setup SB (Sort By) callbacks. @@ -58,7 +62,7 @@ class PackageSearch: "w": self._sort_by_voted, "o": self._sort_by_notify, "m": self._sort_by_maintainer, - "l": self._sort_by_last_modified + "l": self._sort_by_last_modified, } self._joined_user = False @@ -66,12 +70,10 @@ class PackageSearch: self._joined_comaint = False def _join_user(self, outer: bool = True) -> orm.Query: - """ Centralized joining of a package base's maintainer. """ + """Centralized joining of a package base's maintainer.""" if not self._joined_user: self.query = self.query.join( - User, - User.ID == PackageBase.MaintainerUID, - isouter=outer + User, User.ID == PackageBase.MaintainerUID, isouter=outer ) self._joined_user = True return self.query @@ -87,7 +89,7 @@ class PackageSearch: self.query = self.query.join( PackageComaintainer, PackageComaintainer.PackageBaseID == PackageBase.ID, - isouter=isouter + isouter=isouter, ) self._joined_comaint = True return self.query @@ -95,8 +97,10 @@ class PackageSearch: def _search_by_namedesc(self, keywords: str) -> orm.Query: self._join_user() self.query = self.query.filter( - or_(Package.Name.like(f"%{keywords}%"), - Package.Description.like(f"%{keywords}%")) + or_( + Package.Name.like(f"%{keywords}%"), + Package.Description.like(f"%{keywords}%"), + ) ) return self @@ -132,8 +136,7 @@ class PackageSearch: self._join_user() if keywords: self.query = self.query.filter( - and_(User.Username == keywords, - User.ID == PackageBase.MaintainerUID) + and_(User.Username == keywords, User.ID == PackageBase.MaintainerUID) ) else: self.query = self.query.filter(PackageBase.MaintainerUID.is_(None)) @@ -197,8 +200,7 @@ class PackageSearch: # in terms of performance. We should improve this; there's no # reason it should take _longer_. column = getattr( - case([(models.PackageVote.UsersID == self.user.ID, 1)], else_=0), - order + case([(models.PackageVote.UsersID == self.user.ID, 1)], else_=0), order ) name = getattr(models.Package.Name, order) self.query = self.query.order_by(column(), name()) @@ -209,9 +211,8 @@ class PackageSearch: # in terms of performance. We should improve this; there's no # reason it should take _longer_. column = getattr( - case([(models.PackageNotification.UserID == self.user.ID, 1)], - else_=0), - order + case([(models.PackageNotification.UserID == self.user.ID, 1)], else_=0), + order, ) name = getattr(models.Package.Name, order) self.query = self.query.order_by(column(), name()) @@ -239,16 +240,16 @@ class PackageSearch: return callback(ordering) def count(self) -> int: - """ Return internal query's count. """ + """Return internal query's count.""" return self.query.count() def results(self) -> orm.Query: - """ Return internal query. """ + """Return internal query.""" return self.query class RPCSearch(PackageSearch): - """ A PackageSearch-derived RPC package search query builder. + """A PackageSearch-derived RPC package search query builder. With RPC search, we need a subset of PackageSearch's handlers, with a few additional handlers added. So, within the RPCSearch @@ -270,52 +271,60 @@ class RPCSearch(PackageSearch): # We keep: "nd", "n" and "m". We also overlay four new by params # on top: "depends", "makedepends", "optdepends" and "checkdepends". self.search_by_cb = { - k: v for k, v in self.search_by_cb.items() + k: v + for k, v in self.search_by_cb.items() if k not in RPCSearch.keys_removed } - self.search_by_cb.update({ - "depends": self._search_by_depends, - "makedepends": self._search_by_makedepends, - "optdepends": self._search_by_optdepends, - "checkdepends": self._search_by_checkdepends - }) + self.search_by_cb.update( + { + "depends": self._search_by_depends, + "makedepends": self._search_by_makedepends, + "optdepends": self._search_by_optdepends, + "checkdepends": self._search_by_checkdepends, + } + ) # We always want an optional Maintainer in the RPC. self._join_user() def _join_depends(self, dep_type_id: int) -> orm.Query: - """ Join Package with PackageDependency and filter results + """Join Package with PackageDependency and filter results based on `dep_type_id`. :param dep_type_id: DependencyType ID :returns: PackageDependency-joined orm.Query """ self.query = self.query.join(models.PackageDependency).filter( - models.PackageDependency.DepTypeID == dep_type_id) + models.PackageDependency.DepTypeID == dep_type_id + ) return self.query def _search_by_depends(self, keywords: str) -> "RPCSearch": self.query = self._join_depends(DEPENDS_ID).filter( - models.PackageDependency.DepName == keywords) + models.PackageDependency.DepName == keywords + ) return self def _search_by_makedepends(self, keywords: str) -> "RPCSearch": self.query = self._join_depends(MAKEDEPENDS_ID).filter( - models.PackageDependency.DepName == keywords) + models.PackageDependency.DepName == keywords + ) return self def _search_by_optdepends(self, keywords: str) -> "RPCSearch": self.query = self._join_depends(OPTDEPENDS_ID).filter( - models.PackageDependency.DepName == keywords) + models.PackageDependency.DepName == keywords + ) return self def _search_by_checkdepends(self, keywords: str) -> "RPCSearch": self.query = self._join_depends(CHECKDEPENDS_ID).filter( - models.PackageDependency.DepName == keywords) + models.PackageDependency.DepName == keywords + ) return self def search_by(self, by: str, keywords: str) -> "RPCSearch": - """ Override inherited search_by. In this override, we reduce the + """Override inherited search_by. In this override, we reduce the scope of what we handle within this function. We do not set `by` to a default of "nd" in the RPC, as the RPC returns an error when incorrect `by` fields are specified. @@ -329,6 +338,4 @@ class RPCSearch(PackageSearch): return result def results(self) -> orm.Query: - return self.query.filter( - models.PackageBase.PackagerUID.isnot(None) - ) + return self.query.filter(models.PackageBase.PackagerUID.isnot(None)) From 7ddce6bb2d8a18fd9b63a23e7a022197226ef672 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Mon, 14 Mar 2022 05:55:19 -0700 Subject: [PATCH 2/5] doc: update CONTRIBUTING.md Signed-off-by: Kevin Morris --- CONTRIBUTING.md | 74 +++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 72 insertions(+), 2 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 2bb840f5..3d99d887 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -9,7 +9,7 @@ You can add a git hook to do this by installing `python-pre-commit` and running `pre-commit install`. [1]: https://lists.archlinux.org/listinfo/aur-dev -[2]: https://gitlab.archlinunx.org/archlinux/aurweb +[2]: https://gitlab.archlinux.org/archlinux/aurweb ### Coding Guidelines @@ -23,6 +23,76 @@ development. 3. Use four space indentation 4. Use [conventional commits](https://www.conventionalcommits.org/en/v1.0.0/) 5. DRY: Don't Repeat Yourself -6. All code should be tested for good _and_ bad cases +6. All code should be tested for good _and_ bad cases (see [test/README.md][3]) + +[3]: https://gitlab.archlinux.org/archlinux/aurweb/-/blob/master/test/README.md Test patches that increase coverage in the codebase are always welcome. + +### Coding Style + +We use the `flake8` and `isort` tools to manage PEP-8 coherenace and +import ordering in this project. + +There are plugins for editors or IDEs which automate this process. Some +example plugins: + +- [tell-k/vim-autopep8](https://github.com/tell-k/vim-autopep8) +- [fisadev/vim-isort](https://github.com/fisadev/vim-isort) +- [prabirshrestha/vim-lsp](https://github.com/prabirshrestha/vim-lsp) + +See `setup.cfg` for flake8 and isort specific rules. + +Note: We are planning on switching to [psf/black](https://github.com/psf/black). +For now, developers should ensure that flake8 and isort passes when submitting +merge requests or patch sets. + +### Development Environment + +To get started with local development, an instance of aurweb must be +brought up. This can be done using the following sections: + +- [Using Docker](#using-docker) +- [Using INSTALL](#using-install) + +There are a number of services aurweb employs to run the application +in its entirety: + +- ssh +- cron jobs +- starlette/fastapi asgi server + +Project structure: + +- `./aurweb`: `aurweb` Python package +- `./templates`: Jinja2 templates +- `./docker`: Docker scripts and configuration files + +#### Using Docker + +Using Docker, we can run the entire infrastructure in two steps: + + # Build the aurweb:latest image + $ docker-compose build + + # Start all services in the background + $ docker-compose up -d nginx + +`docker-compose` services will generate a locally signed root certificate +at `./data/root_ca.crt`. Users can import this into ca-certificates or their +browser if desired. + +Accessible services (on the host): + +- https://localhost:8444 (python via nginx) +- https://localhost:8443 (php via nginx) +- localhost:13306 (mariadb) +- localhost:16379 (redis) + +Docker services, by default, are setup to be hot reloaded when source code +is changed. + +#### Using INSTALL + +The [INSTALL](INSTALL) file describes steps to install the application on +bare-metal systems. From 790ca4194a6360e9f47f56fe9d39aae4cbe14c25 Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Mon, 14 Mar 2022 05:57:06 -0700 Subject: [PATCH 3/5] fix: coherenace -> coherence Signed-off-by: Kevin Morris --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 3d99d887..52e182c7 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -31,7 +31,7 @@ Test patches that increase coverage in the codebase are always welcome. ### Coding Style -We use the `flake8` and `isort` tools to manage PEP-8 coherenace and +We use the `flake8` and `isort` tools to manage PEP-8 coherence and import ordering in this project. There are plugins for editors or IDEs which automate this process. Some From afd25c248fcee508da6724398f6c37c47bf4be5e Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Mon, 14 Mar 2022 06:24:15 -0700 Subject: [PATCH 4/5] fix: remove HEAD and OPTIONS handling from metrics Signed-off-by: Kevin Morris --- aurweb/prometheus.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/aurweb/prometheus.py b/aurweb/prometheus.py index 73be3ef6..272ee023 100644 --- a/aurweb/prometheus.py +++ b/aurweb/prometheus.py @@ -60,6 +60,9 @@ def http_requests_total() -> Callable[[Info], None]: labelnames=("method", "path", "status")) def instrumentation(info: Info) -> None: + if info.request.method.lower() in ("head", "options"): # pragma: no cover + return + scope = info.request.scope # Taken from https://github.com/stephenhillier/starlette_exporter @@ -70,8 +73,8 @@ def http_requests_total() -> Callable[[Info], None]: if not (scope.get("endpoint", None) and scope.get("router", None)): return None - root_path = scope.get("root_path", "") - app = scope.get("app", {}) + root_path = scope.get("root_path", str()) + app = scope.get("app", dict()) if hasattr(app, "root_path"): app_root_path = getattr(app, "root_path") @@ -102,6 +105,9 @@ def http_api_requests_total() -> Callable[[Info], None]: labelnames=("type", "status")) def instrumentation(info: Info) -> None: + if info.request.method.lower() in ("head", "options"): # pragma: no cover + return + if info.request.url.path.rstrip("/") == "/rpc": type = info.request.query_params.get("type", "None") if info.response: From d8564e446b744bbc7b6bd8fea22ab6b614acc5ab Mon Sep 17 00:00:00 2001 From: Kevin Morris Date: Wed, 30 Mar 2022 12:30:21 -0700 Subject: [PATCH 5/5] upgrade: bump to v6.0.26 Signed-off-by: Kevin Morris --- aurweb/config.py | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/aurweb/config.py b/aurweb/config.py index 9565b70c..53942b75 100644 --- a/aurweb/config.py +++ b/aurweb/config.py @@ -6,7 +6,7 @@ from typing import Any # Publicly visible version of aurweb. This is used to display # aurweb versioning in the footer and must be maintained. # Todo: Make this dynamic/automated. -AURWEB_VERSION = "v6.0.25" +AURWEB_VERSION = "v6.0.26" _parser = None diff --git a/pyproject.toml b/pyproject.toml index 8b7a2e93..b15af272 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,7 +8,7 @@ # [tool.poetry] name = "aurweb" -version = "v6.0.25" +version = "v6.0.26" license = "GPL-2.0-only" description = "Source code for the Arch User Repository's website" homepage = "https://aur.archlinux.org"