diff --git a/aurweb/scripts/rendercomment.py b/aurweb/scripts/rendercomment.py index efa5357f..33349432 100755 --- a/aurweb/scripts/rendercomment.py +++ b/aurweb/scripts/rendercomment.py @@ -7,13 +7,11 @@ import markdown import pygit2 import aurweb.config -import aurweb.db -from aurweb import logging +from aurweb import db, logging, util +from aurweb.models import PackageComment logger = logging.get_logger(__name__) -repo_path = aurweb.config.get('serve', 'repo-path') -commit_uri = aurweb.config.get('options', 'commit_uri') class LinkifyExtension(markdown.extensions.Extension): @@ -64,6 +62,7 @@ class GitCommitsInlineProcessor(markdown.inlinepatterns.InlineProcessor): """ def __init__(self, md, head): + repo_path = aurweb.config.get('serve', 'repo-path') self._repo = pygit2.Repository(repo_path) self._head = head super().__init__(r'\b([0-9a-f]{7,40})\b', md) @@ -74,13 +73,9 @@ class GitCommitsInlineProcessor(markdown.inlinepatterns.InlineProcessor): # Unkwown OID; preserve the orginal text. return (None, None, None) - prefixlen = 12 - while prefixlen < 40: - if oid[:prefixlen] in self._repo: - break - prefixlen += 1 - el = markdown.util.etree.Element('a') + commit_uri = aurweb.config.get("options", "commit_uri") + prefixlen = util.git_search(self._repo, oid) el.set('href', commit_uri % (self._head, oid[:prefixlen])) el.text = markdown.util.AtomicString(oid[:prefixlen]) return (el, m.start(0), m.end(0)) @@ -116,49 +111,41 @@ class HeadingExtension(markdown.extensions.Extension): md.treeprocessors.register(HeadingTreeprocessor(md), 'heading', 30) -def get_comment(conn, commentid): - cur = conn.execute('SELECT PackageComments.Comments, PackageBases.Name ' - 'FROM PackageComments INNER JOIN PackageBases ' - 'ON PackageBases.ID = PackageComments.PackageBaseID ' - 'WHERE PackageComments.ID = ?', [commentid]) - return cur.fetchone() +def save_rendered_comment(comment: PackageComment, html: str): + with db.begin(): + comment.RenderedComment = html -def save_rendered_comment(conn, commentid, html): - conn.execute('UPDATE PackageComments SET RenderedComment = ? WHERE ID = ?', - [html, commentid]) +def update_comment_render_fastapi(comment: PackageComment) -> None: + update_comment_render(comment) -def update_comment_render_fastapi(comment): - conn = aurweb.db.ConnectionExecutor( - aurweb.db.get_engine().raw_connection()) - update_comment_render(conn, comment.ID) - aurweb.db.refresh(comment) +def update_comment_render(comment: PackageComment) -> None: + text = comment.Comments + pkgbasename = comment.PackageBase.Name - -def update_comment_render(conn, commentid): - text, pkgbase = get_comment(conn, commentid) html = markdown.markdown(text, extensions=[ 'fenced_code', LinkifyExtension(), FlysprayLinksExtension(), - GitCommitsExtension(pkgbase), + GitCommitsExtension(pkgbasename), HeadingExtension() ]) allowed_tags = (bleach.sanitizer.ALLOWED_TAGS + ['p', 'pre', 'h4', 'h5', 'h6', 'br', 'hr']) html = bleach.clean(html, tags=allowed_tags) - save_rendered_comment(conn, commentid, html) - - conn.commit() - conn.close() + save_rendered_comment(comment, html) + db.refresh(comment) def main(): - commentid = int(sys.argv[1]) - conn = aurweb.db.Connection() - update_comment_render(conn, commentid) + db.get_engine() + comment_id = int(sys.argv[1]) + comment = db.query(PackageComment).filter( + PackageComment.ID == comment_id + ).first() + update_comment_render(comment) if __name__ == '__main__': diff --git a/aurweb/testing/git.py b/aurweb/testing/git.py new file mode 100644 index 00000000..019d870f --- /dev/null +++ b/aurweb/testing/git.py @@ -0,0 +1,110 @@ +import os +import shlex + +from subprocess import PIPE, Popen +from typing import Tuple + +import py + +from aurweb.models import Package +from aurweb.templates import base_template +from aurweb.testing.filelock import FileLock + + +class GitRepository: + """ + A Git repository class to be used for testing. + + Expects a `tmpdir` fixture on construction, which an 'aur.git' + git repository will be created in. After this class is constructed, + users can call GitRepository.exec for git repository operations. + """ + + def __init__(self, tmpdir: py.path.local): + self.file_lock = FileLock(tmpdir, "aur.git") + self.file_lock.lock(on_create=self._setup) + + def _exec(self, cmdline: str, cwd: str) -> Tuple[int, str, str]: + args = shlex.split(cmdline) + proc = Popen(args, cwd=cwd, stdout=PIPE, stderr=PIPE) + out, err = proc.communicate() + return (proc.returncode, out.decode().strip(), err.decode().strip()) + + def _exec_repository(self, cmdline: str) -> Tuple[int, str, str]: + return self._exec(cmdline, cwd=str(self.file_lock.path)) + + def exec(self, cmdline: str) -> Tuple[int, str, str]: + return self._exec_repository(cmdline) + + def _setup(self, path: str) -> None: + """ + Setup the git repository from scratch. + + Create the `path` directory and run the INSTALL recommended + git initialization commands inside of it. Additionally, install + aurweb.git.update to {path}/hooks/update. + + :param path: Repository path not yet created + """ + + os.makedirs(path) + + commands = [ + "git init -q", + "git config --local transfer.hideRefs '^refs/'", + "git config --local --add transfer.hideRefs '!refs/'", + "git config --local --add transfer.hideRefs '!HEAD'", + "git config --local commit.gpgsign false", + "git config --local user.name 'Test User'", + "git config --local user.email 'test@example.org'", + ] + for cmdline in commands: + return_code, out, err = self.exec(cmdline) + assert return_code == 0 + + # This is also done in the INSTALL script to give the `aur` + # ssh user permissions on the repository. We don't need it + # during testing, since our testing user will be controlling + # the repository. It is left here as a note. + # self.exec("chown -R aur .") + + def commit(self, pkg: Package, message: str): + """ + Commit a Package record to the git repository. + + This function generates a PKGBUILD and .SRCINFO based on + `pkg`, then commits them to the repository with the + `message` commit message. + + :param pkg: Package instance + :param message: Commit message + :return: Output of `git rev-parse HEAD` after committing + """ + ref = f"refs/namespaces/{pkg.Name}/refs/heads/master" + rc, out, err = self.exec(f"git checkout -q --orphan {ref}") + assert rc == 0, f"{(rc, out, err)}" + + # Path to aur.git repository. + repo = os.path.join(self.file_lock.path) + + licenses = [f"'{p.License.Name}'" for p in pkg.package_licenses] + depends = [f"'{p.DepName}'" for p in pkg.package_dependencies] + pkgbuild = base_template("testing/PKGBUILD.j2") + pkgbuild_path = os.path.join(repo, "PKGBUILD") + with open(pkgbuild_path, "w") as f: + data = pkgbuild.render(pkg=pkg, licenses=licenses, depends=depends) + f.write(data) + + srcinfo = base_template("testing/SRCINFO.j2") + srcinfo_path = os.path.join(repo, ".SRCINFO") + with open(srcinfo_path, "w") as f: + f.write(srcinfo.render(pkg=pkg)) + + rc, out, err = self.exec("git add PKGBUILD .SRCINFO") + assert rc == 0, f"{(rc, out, err)}" + + rc, out, err = self.exec(f"git commit -q -m '{message}'") + assert rc == 0, f"{(rc, out, err)}" + + # Return stdout of `git rev-parse HEAD`, which is the new commit hash. + return self.exec("git rev-parse HEAD")[1] diff --git a/aurweb/util.py b/aurweb/util.py index bf2d6e4b..f5ced259 100644 --- a/aurweb/util.py +++ b/aurweb/util.py @@ -13,6 +13,7 @@ from urllib.parse import urlencode, urlparse from zoneinfo import ZoneInfo import fastapi +import pygit2 from email_validator import EmailNotValidError, EmailUndeliverableError, validate_email from jinja2 import pass_context @@ -193,3 +194,19 @@ def file_hash(filepath: str, hash_function: Callable) -> str: with open(filepath, "rb") as f: hash_ = hash_function(f.read()) return hash_.hexdigest() + + +def git_search(repo: pygit2.Repository, commit_hash: str) -> int: + """ + Return the shortest prefix length matching `commit_hash` found. + + :param repo: pygit2.Repository instance + :param commit_hash: Full length commit hash + :return: Shortest unique prefix length found + """ + prefixlen = 12 + while prefixlen < len(commit_hash): + if commit_hash[:prefixlen] in repo: + break + prefixlen += 1 + return prefixlen diff --git a/templates/testing/PKGBUILD.j2 b/templates/testing/PKGBUILD.j2 new file mode 100644 index 00000000..29d3a1d9 --- /dev/null +++ b/templates/testing/PKGBUILD.j2 @@ -0,0 +1,14 @@ +pkgname={{ pkg.PackageBase.Name }} +pkgver={{ pkg.Version }} +pkgrel=1 +pkgdesc='{{ pkg.Description }}' +url='{{ pkg.URL }}' +arch='any' +license=({{ licenses | join(" ") }}) +depends=({{ depends | join(" ") }}) +source=() +md5sums=() + +package() { + {{ body }} +} diff --git a/templates/testing/SRCINFO.j2 b/templates/testing/SRCINFO.j2 new file mode 100644 index 00000000..873b9c1b --- /dev/null +++ b/templates/testing/SRCINFO.j2 @@ -0,0 +1,10 @@ +pkgbase = {{ pkg.PackageBase.name }} + pkgver = {{ pkg.Version }} + pkgrel = 1 + pkgdesc = {{ pkg.Description }} + url = {{ pkg.URL }} + arch='any' + license = {{ pkg.package_licenses | join(", ", attribute="License.Name") }} + depends = {{ pkg.package_dependencies | join(", ", attribute="DepName") }} + +pkgname = {{ pkg.Name }} diff --git a/test/conftest.py b/test/conftest.py index 80f77c9a..fc7f77dc 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -58,6 +58,7 @@ import aurweb.db from aurweb import initdb, logging, testing from aurweb.testing.email import Email from aurweb.testing.filelock import FileLock +from aurweb.testing.git import GitRepository logger = logging.get_logger(__name__) @@ -211,3 +212,8 @@ def db_test(db_session: scoped_session) -> None: session via aurweb.db.get_session(). """ testing.setup_test_db() + + +@pytest.fixture +def git(tmpdir: py.path.local) -> GitRepository: + yield GitRepository(tmpdir) diff --git a/test/t2600-rendercomment.t b/test/t2600-rendercomment.t deleted file mode 100755 index bb84fcfe..00000000 --- a/test/t2600-rendercomment.t +++ /dev/null @@ -1,160 +0,0 @@ -#!/bin/sh - -test_description='rendercomment tests' - -. "$(dirname "$0")/setup.sh" - -test_expect_success 'Test comment rendering.' ' - cat <<-EOD | sqlite3 aur.db && - INSERT INTO PackageBases (ID, Name, PackagerUID, SubmittedTS, ModifiedTS, FlaggerComment) VALUES (1, "foobar", 1, 0, 0, ""); - INSERT INTO PackageComments (ID, PackageBaseID, Comments, RenderedComment) VALUES (1, 1, "Hello world! - This is a comment.", ""); - EOD - cover "$RENDERCOMMENT" 1 && - cat <<-EOD >expected && -

Hello world! - This is a comment.

- EOD - cat <<-EOD | sqlite3 aur.db >actual && - SELECT RenderedComment FROM PackageComments WHERE ID = 1; - EOD - test_cmp actual expected -' - -test_expect_success 'Test Markdown conversion.' ' - cat <<-EOD | sqlite3 aur.db && - INSERT INTO PackageComments (ID, PackageBaseID, Comments, RenderedComment) VALUES (2, 1, "*Hello* [world](https://www.archlinux.org/)!", ""); - EOD - cover "$RENDERCOMMENT" 2 && - cat <<-EOD >expected && -

Hello world!

- EOD - cat <<-EOD | sqlite3 aur.db >actual && - SELECT RenderedComment FROM PackageComments WHERE ID = 2; - EOD - test_cmp actual expected -' - -test_expect_success 'Test HTML sanitizing.' ' - cat <<-EOD | sqlite3 aur.db && - INSERT INTO PackageComments (ID, PackageBaseID, Comments, RenderedComment) VALUES (3, 1, "", ""); - EOD - cover "$RENDERCOMMENT" 3 && - cat <<-EOD >expected && - <script>alert("XSS!");</script> - EOD - cat <<-EOD | sqlite3 aur.db >actual && - SELECT RenderedComment FROM PackageComments WHERE ID = 3; - EOD - test_cmp actual expected -' - -test_expect_success 'Test link conversion.' ' - cat <<-EOD | sqlite3 aur.db && - INSERT INTO PackageComments (ID, PackageBaseID, Comments, RenderedComment) VALUES (4, 1, " - Visit https://www.archlinux.org/#_test_. - Visit *https://www.archlinux.org/*. - Visit . - Visit \`https://www.archlinux.org/\`. - Visit [Arch Linux](https://www.archlinux.org/). - Visit [Arch Linux][arch]. - [arch]: https://www.archlinux.org/ - ", ""); - EOD - cover "$RENDERCOMMENT" 4 && - cat <<-EOD >expected && -

Visit https://www.archlinux.org/#_test_. - Visit https://www.archlinux.org/. - Visit https://www.archlinux.org/. - Visit https://www.archlinux.org/. - Visit Arch Linux. - Visit Arch Linux.

- EOD - cat <<-EOD | sqlite3 aur.db >actual && - SELECT RenderedComment FROM PackageComments WHERE ID = 4; - EOD - test_cmp actual expected -' - -test_expect_success 'Test Git commit linkification.' ' - local oid=`git -C aur.git rev-parse --verify HEAD` - cat <<-EOD | sqlite3 aur.db && - INSERT INTO PackageComments (ID, PackageBaseID, Comments, RenderedComment) VALUES (5, 1, " - $oid - ${oid:0:7} - x.$oid.x - ${oid}x - 0123456789abcdef - \`$oid\` - http://example.com/$oid - ", ""); - EOD - cover "$RENDERCOMMENT" 5 && - cat <<-EOD >expected && -

${oid:0:12} - ${oid:0:7} - x.${oid:0:12}.x - ${oid}x - 0123456789abcdef - $oid - http://example.com/$oid

- EOD - cat <<-EOD | sqlite3 aur.db >actual && - SELECT RenderedComment FROM PackageComments WHERE ID = 5; - EOD - test_cmp actual expected -' - -test_expect_success 'Test Flyspray issue linkification.' ' - sqlite3 aur.db <<-EOD && - INSERT INTO PackageComments (ID, PackageBaseID, Comments, RenderedComment) VALUES (6, 1, " - FS#1234567. - *FS#1234* - FS# - XFS#1 - \`FS#1234\` - https://archlinux.org/?test=FS#1234 - ", ""); - EOD - cover "$RENDERCOMMENT" 6 && - cat <<-EOD >expected && -

FS#1234567. - FS#1234 - FS# - XFS#1 - FS#1234 - https://archlinux.org/?test=FS#1234

- EOD - sqlite3 aur.db <<-EOD >actual && - SELECT RenderedComment FROM PackageComments WHERE ID = 6; - EOD - test_cmp actual expected -' - -test_expect_success 'Test headings lowering.' ' - sqlite3 aur.db <<-EOD && - INSERT INTO PackageComments (ID, PackageBaseID, Comments, RenderedComment) VALUES (7, 1, " - # One - ## Two - ### Three - #### Four - ##### Five - ###### Six - ", ""); - EOD - cover "$RENDERCOMMENT" 7 && - cat <<-EOD >expected && -
One
-
Two
-
Three
-
Four
-
Five
-
Six
- EOD - sqlite3 aur.db <<-EOD >actual && - SELECT RenderedComment FROM PackageComments WHERE ID = 7; - EOD - test_cmp actual expected -' - -test_done diff --git a/test/test_rendercomment.py b/test/test_rendercomment.py new file mode 100644 index 00000000..c45d4235 --- /dev/null +++ b/test/test_rendercomment.py @@ -0,0 +1,202 @@ +from datetime import datetime +from unittest import mock + +import pytest + +from aurweb import config, db, logging +from aurweb.models import Package, PackageBase, PackageComment, User +from aurweb.models.account_type import USER_ID +from aurweb.scripts import rendercomment +from aurweb.scripts.rendercomment import update_comment_render +from aurweb.testing.git import GitRepository + +logger = logging.get_logger(__name__) +aur_location = config.get("options", "aur_location") + + +@pytest.fixture(autouse=True) +def setup(db_test, git: GitRepository): + config_get = config.get + + def mock_config_get(section: str, key: str) -> str: + if section == "serve" and key == "repo-path": + return git.file_lock.path + elif section == "options" and key == "commit_uri": + return "/cgit/aur.git/log/?h=%s&id=%s" + return config_get(section, key) + + with mock.patch("aurweb.config.get", side_effect=mock_config_get): + yield + + +@pytest.fixture +def user() -> User: + with db.begin(): + user = db.create(User, Username="test", Email="test@example.org", + Passwd=str(), AccountTypeID=USER_ID) + yield user + + +@pytest.fixture +def pkgbase(user: User) -> PackageBase: + now = int(datetime.utcnow().timestamp()) + with db.begin(): + pkgbase = db.create(PackageBase, Packager=user, Name="pkgbase_0", + SubmittedTS=now, ModifiedTS=now) + yield pkgbase + + +@pytest.fixture +def package(pkgbase: PackageBase) -> Package: + with db.begin(): + package = db.create(Package, PackageBase=pkgbase, + Name=pkgbase.Name, Version="1.0") + yield package + + +def create_comment(user: User, pkgbase: PackageBase, comments: str, + render: bool = True): + with db.begin(): + comment = db.create(PackageComment, User=user, + PackageBase=pkgbase, Comments=comments) + if render: + update_comment_render(comment) + return comment + + +def test_comment_rendering(user: User, pkgbase: PackageBase): + text = "Hello world! This is a comment." + comment = create_comment(user, pkgbase, text) + expected = f"

{text}

" + assert comment.RenderedComment == expected + + +def test_rendercomment_main(user: User, pkgbase: PackageBase): + text = "Hello world! This is a comment." + comment = create_comment(user, pkgbase, text, False) + + args = ["aurweb-rendercomment", str(comment.ID)] + with mock.patch("sys.argv", args): + rendercomment.main() + db.refresh(comment) + + expected = f"

{text}

" + assert comment.RenderedComment == expected + + +def test_markdown_conversion(user: User, pkgbase: PackageBase): + text = "*Hello* [world](https://aur.archlinux.org)!" + comment = create_comment(user, pkgbase, text) + expected = ('

Hello ' + 'world!

') + assert comment.RenderedComment == expected + + +def test_html_sanitization(user: User, pkgbase: PackageBase): + text = '' + comment = create_comment(user, pkgbase, text) + expected = '<script>alert("XSS!")</script>' + assert comment.RenderedComment == expected + + +def test_link_conversion(user: User, pkgbase: PackageBase): + text = """\ +Visit https://www.archlinux.org/#_test_. +Visit *https://www.archlinux.org/*. +Visit . +Visit `https://www.archlinux.org/`. +Visit [Arch Linux](https://www.archlinux.org/). +Visit [Arch Linux][arch]. +[arch]: https://www.archlinux.org/\ +""" + comment = create_comment(user, pkgbase, text) + expected = '''\ +

Visit \ +https://www.archlinux.org/#_test_. +Visit https://www.archlinux.org/. +Visit https://www.archlinux.org/. +Visit https://www.archlinux.org/. +Visit Arch Linux. +Visit Arch Linux.

\ +''' + assert comment.RenderedComment == expected + + +def test_git_commit_link(git: GitRepository, user: User, package: Package): + commit_hash = git.commit(package, "Initial commit.") + logger.info(f"Created commit: {commit_hash}") + logger.info(f"Short hash: {commit_hash[:7]}") + + text = f"""\ +{commit_hash} +{commit_hash[:7]} +x.{commit_hash}.x +{commit_hash}x +0123456789abcdef +`{commit_hash}` +http://example.com/{commit_hash}\ +""" + comment = create_comment(user, package.PackageBase, text) + + pkgname = package.PackageBase.Name + cgit_path = f"/cgit/aur.git/log/?h={pkgname}&" + expected = f"""\ +

{commit_hash[:12]} +{commit_hash[:7]} +x.{commit_hash[:12]}.x +{commit_hash}x +0123456789abcdef +{commit_hash} +\ +http://example.com/{commit_hash}\ +\ +

\ +""" + assert comment.RenderedComment == expected + + +def test_flyspray_issue_link(user: User, pkgbase: PackageBase): + text = """\ +FS#1234567. +*FS#1234* +FS# +XFS#1 +`FS#1234` +https://archlinux.org/?test=FS#1234\ +""" + comment = create_comment(user, pkgbase, text) + + expected = """\ +

FS#1234567. +FS#1234 +FS# +XFS#1 +FS#1234 +\ +https://archlinux.org/?test=FS#1234\ +\ +

\ +""" + assert comment.RenderedComment == expected + + +def test_lower_headings(user: User, pkgbase: PackageBase): + text = """\ +# One +## Two +### Three +#### Four +##### Five +###### Six\ +""" + comment = create_comment(user, pkgbase, text) + + expected = """\ +
One
+
Two
+
Three
+
Four
+
Five
+
Six
\ +""" + assert comment.RenderedComment == expected diff --git a/test/test_util.py b/test/test_util.py index 99b77a78..2529ed1f 100644 --- a/test/test_util.py +++ b/test/test_util.py @@ -40,3 +40,20 @@ def test_round(): assert filters.do_round(1.3) == 1 assert filters.do_round(1.5) == 2 assert filters.do_round(2.0) == 2 + + +def test_git_search(): + """ Test that git_search matches the full commit if necessary. """ + commit_hash = "0123456789abcdef" + repo = {commit_hash} + prefixlen = util.git_search(repo, commit_hash) + assert prefixlen == 16 + + +def test_git_search_double_commit(): + """ Test that git_search matches a shorter prefix length. """ + commit_hash = "0123456789abcdef" + repo = {commit_hash[:13]} + # Locate the shortest prefix length that matches commit_hash. + prefixlen = util.git_search(repo, commit_hash) + assert prefixlen == 13