external/aurweb
0
0
Fork 0
mirror of https://gitlab.archlinux.org/archlinux/aurweb.git synced 2025-02-03 10:43:03 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions

change(rendercomment): converted to use aurweb.db ORM

Browse source 
- Added aurweb.util.git_search.
    - Decoupled away from rendercomment for easier testability.
- Added aurweb.testing.git.GitRepository.
- Added templates/testing/{PKGBUILD,SRCINFO}.j2.
- Added aurweb.testing.git.GitRepository + `git` pytest fixture

Signed-off-by: Kevin Morris <kevr@0cost.org>
This commit is contained in:
Kevin Morris 2021-11-17 06:20:50 -08:00
parent 4b0cb0721d
commit 2d0e09cd63
No known key found for this signature in database
GPG key ID: F7E46DED420788F3
9 changed files with 398 additions and 195 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

57
aurweb/scripts/rendercomment.py
View file

@ -7,13 +7,11 @@ import markdown
import pygit2
import aurweb.config
import aurweb.db
from aurweb import logging
from aurweb import db, logging, util
from aurweb.models import PackageComment
logger = logging.get_logger(__name__)
repo_path = aurweb.config.get('serve', 'repo-path')
commit_uri = aurweb.config.get('options', 'commit_uri')
class LinkifyExtension(markdown.extensions.Extension):
@ -64,6 +62,7 @@ class GitCommitsInlineProcessor(markdown.inlinepatterns.InlineProcessor):
"""
def __init__(self, md, head):
repo_path = aurweb.config.get('serve', 'repo-path')
self._repo = pygit2.Repository(repo_path)
self._head = head
super().__init__(r'\b([0-9a-f]{7,40})\b', md)
@ -74,13 +73,9 @@ class GitCommitsInlineProcessor(markdown.inlinepatterns.InlineProcessor):
# Unkwown OID; preserve the orginal text.
return (None, None, None)
prefixlen = 12
while prefixlen < 40:
if oid[:prefixlen] in self._repo:
break
prefixlen += 1
el = markdown.util.etree.Element('a')
commit_uri = aurweb.config.get("options", "commit_uri")
prefixlen = util.git_search(self._repo, oid)
el.set('href', commit_uri % (self._head, oid[:prefixlen]))
el.text = markdown.util.AtomicString(oid[:prefixlen])
return (el, m.start(0), m.end(0))
@ -116,49 +111,41 @@ class HeadingExtension(markdown.extensions.Extension):
md.treeprocessors.register(HeadingTreeprocessor(md), 'heading', 30)
def get_comment(conn, commentid):
cur = conn.execute('SELECT PackageComments.Comments, PackageBases.Name '
'FROM PackageComments INNER JOIN PackageBases '
'ON PackageBases.ID = PackageComments.PackageBaseID '
'WHERE PackageComments.ID = ?', [commentid])
return cur.fetchone()
def save_rendered_comment(comment: PackageComment, html: str):
with db.begin():
comment.RenderedComment = html
def save_rendered_comment(conn, commentid, html):
conn.execute('UPDATE PackageComments SET RenderedComment = ? WHERE ID = ?',
[html, commentid])
def update_comment_render_fastapi(comment: PackageComment) -> None:
update_comment_render(comment)
def update_comment_render_fastapi(comment):
conn = aurweb.db.ConnectionExecutor(
aurweb.db.get_engine().raw_connection())
update_comment_render(conn, comment.ID)
aurweb.db.refresh(comment)
def update_comment_render(comment: PackageComment) -> None:
text = comment.Comments
pkgbasename = comment.PackageBase.Name
def update_comment_render(conn, commentid):
text, pkgbase = get_comment(conn, commentid)
html = markdown.markdown(text, extensions=[
'fenced_code',
LinkifyExtension(),
FlysprayLinksExtension(),
GitCommitsExtension(pkgbase),
GitCommitsExtension(pkgbasename),
HeadingExtension()
])
allowed_tags = (bleach.sanitizer.ALLOWED_TAGS
+ ['p', 'pre', 'h4', 'h5', 'h6', 'br', 'hr'])
html = bleach.clean(html, tags=allowed_tags)
save_rendered_comment(conn, commentid, html)
conn.commit()
conn.close()
save_rendered_comment(comment, html)
db.refresh(comment)
def main():
commentid = int(sys.argv[1])
conn = aurweb.db.Connection()
update_comment_render(conn, commentid)
db.get_engine()
comment_id = int(sys.argv[1])
comment = db.query(PackageComment).filter(
PackageComment.ID == comment_id
).first()
update_comment_render(comment)
if __name__ == '__main__':

110
aurweb/testing/git.py Normal file
View file

@ -0,0 +1,110 @@
import os
import shlex
from subprocess import PIPE, Popen
from typing import Tuple
import py
from aurweb.models import Package
from aurweb.templates import base_template
from aurweb.testing.filelock import FileLock
class GitRepository:
"""
A Git repository class to be used for testing.
Expects a `tmpdir` fixture on construction, which an 'aur.git'
git repository will be created in. After this class is constructed,
users can call GitRepository.exec for git repository operations.
"""
def __init__(self, tmpdir: py.path.local):
self.file_lock = FileLock(tmpdir, "aur.git")
self.file_lock.lock(on_create=self._setup)
def _exec(self, cmdline: str, cwd: str) -> Tuple[int, str, str]:
args = shlex.split(cmdline)
proc = Popen(args, cwd=cwd, stdout=PIPE, stderr=PIPE)
out, err = proc.communicate()
return (proc.returncode, out.decode().strip(), err.decode().strip())
def _exec_repository(self, cmdline: str) -> Tuple[int, str, str]:
return self._exec(cmdline, cwd=str(self.file_lock.path))
def exec(self, cmdline: str) -> Tuple[int, str, str]:
return self._exec_repository(cmdline)
def _setup(self, path: str) -> None:
"""
Setup the git repository from scratch.
Create the `path` directory and run the INSTALL recommended
git initialization commands inside of it. Additionally, install
aurweb.git.update to {path}/hooks/update.
:param path: Repository path not yet created
"""
os.makedirs(path)
commands = [
"git init -q",
"git config --local transfer.hideRefs '^refs/'",
"git config --local --add transfer.hideRefs '!refs/'",
"git config --local --add transfer.hideRefs '!HEAD'",
"git config --local commit.gpgsign false",
"git config --local user.name 'Test User'",
"git config --local user.email 'test@example.org'",
]
for cmdline in commands:
return_code, out, err = self.exec(cmdline)
assert return_code == 0
# This is also done in the INSTALL script to give the `aur`
# ssh user permissions on the repository. We don't need it
# during testing, since our testing user will be controlling
# the repository. It is left here as a note.
# self.exec("chown -R aur .")
def commit(self, pkg: Package, message: str):
"""
Commit a Package record to the git repository.
This function generates a PKGBUILD and .SRCINFO based on
`pkg`, then commits them to the repository with the
`message` commit message.
:param pkg: Package instance
:param message: Commit message
:return: Output of `git rev-parse HEAD` after committing
"""
ref = f"refs/namespaces/{pkg.Name}/refs/heads/master"
rc, out, err = self.exec(f"git checkout -q --orphan {ref}")
assert rc == 0, f"{(rc, out, err)}"
# Path to aur.git repository.
repo = os.path.join(self.file_lock.path)
licenses = [f"'{p.License.Name}'" for p in pkg.package_licenses]
depends = [f"'{p.DepName}'" for p in pkg.package_dependencies]
pkgbuild = base_template("testing/PKGBUILD.j2")
pkgbuild_path = os.path.join(repo, "PKGBUILD")
with open(pkgbuild_path, "w") as f:
data = pkgbuild.render(pkg=pkg, licenses=licenses, depends=depends)
f.write(data)
srcinfo = base_template("testing/SRCINFO.j2")
srcinfo_path = os.path.join(repo, ".SRCINFO")
with open(srcinfo_path, "w") as f:
f.write(srcinfo.render(pkg=pkg))
rc, out, err = self.exec("git add PKGBUILD .SRCINFO")
assert rc == 0, f"{(rc, out, err)}"
rc, out, err = self.exec(f"git commit -q -m '{message}'")
assert rc == 0, f"{(rc, out, err)}"
# Return stdout of `git rev-parse HEAD`, which is the new commit hash.
return self.exec("git rev-parse HEAD")[1]

17
aurweb/util.py
View file

@ -13,6 +13,7 @@ from urllib.parse import urlencode, urlparse
from zoneinfo import ZoneInfo
import fastapi
import pygit2
from email_validator import EmailNotValidError, EmailUndeliverableError, validate_email
from jinja2 import pass_context
@ -193,3 +194,19 @@ def file_hash(filepath: str, hash_function: Callable) -> str:
with open(filepath, "rb") as f:
hash_ = hash_function(f.read())
return hash_.hexdigest()
def git_search(repo: pygit2.Repository, commit_hash: str) -> int:
"""
Return the shortest prefix length matching `commit_hash` found.
:param repo: pygit2.Repository instance
:param commit_hash: Full length commit hash
:return: Shortest unique prefix length found
"""
prefixlen = 12
while prefixlen < len(commit_hash):
if commit_hash[:prefixlen] in repo:
break
prefixlen += 1
return prefixlen

14
templates/testing/PKGBUILD.j2 Normal file
View file

@ -0,0 +1,14 @@
pkgname={{ pkg.PackageBase.Name }}
pkgver={{ pkg.Version }}
pkgrel=1
pkgdesc='{{ pkg.Description }}'
url='{{ pkg.URL }}'
arch='any'
license=({{ licenses | join(" ") }})
depends=({{ depends | join(" ") }})
source=()
md5sums=()
package() {
{{ body }}
}

10
templates/testing/SRCINFO.j2 Normal file
View file

@ -0,0 +1,10 @@
pkgbase = {{ pkg.PackageBase.name }}
pkgver = {{ pkg.Version }}
pkgrel = 1
pkgdesc = {{ pkg.Description }}
url = {{ pkg.URL }}
arch='any'
license = {{ pkg.package_licenses | join(", ", attribute="License.Name") }}
depends = {{ pkg.package_dependencies | join(", ", attribute="DepName") }}
pkgname = {{ pkg.Name }}

6
test/conftest.py
View file

@ -58,6 +58,7 @@ import aurweb.db
from aurweb import initdb, logging, testing
from aurweb.testing.email import Email
from aurweb.testing.filelock import FileLock
from aurweb.testing.git import GitRepository
logger = logging.get_logger(__name__)
@ -211,3 +212,8 @@ def db_test(db_session: scoped_session) -> None:
session via aurweb.db.get_session().
"""
testing.setup_test_db()
@pytest.fixture
def git(tmpdir: py.path.local) -> GitRepository:
yield GitRepository(tmpdir)

160
test/t2600-rendercomment.t
View file

@ -1,160 +0,0 @@
#!/bin/sh
test_description='rendercomment tests'
. "$(dirname "$0")/setup.sh"
test_expect_success 'Test comment rendering.' '
cat <<-EOD | sqlite3 aur.db &&
INSERT INTO PackageBases (ID, Name, PackagerUID, SubmittedTS, ModifiedTS, FlaggerComment) VALUES (1, "foobar", 1, 0, 0, "");
INSERT INTO PackageComments (ID, PackageBaseID, Comments, RenderedComment) VALUES (1, 1, "Hello world!
This is a comment.", "");
EOD
cover "$RENDERCOMMENT" 1 &&
cat <<-EOD >expected &&
<p>Hello world!
This is a comment.</p>
EOD
cat <<-EOD | sqlite3 aur.db >actual &&
SELECT RenderedComment FROM PackageComments WHERE ID = 1;
EOD
test_cmp actual expected
'
test_expect_success 'Test Markdown conversion.' '
cat <<-EOD | sqlite3 aur.db &&
INSERT INTO PackageComments (ID, PackageBaseID, Comments, RenderedComment) VALUES (2, 1, "*Hello* [world](https://www.archlinux.org/)!", "");
EOD
cover "$RENDERCOMMENT" 2 &&
cat <<-EOD >expected &&
<p><em>Hello</em> <a href="https://www.archlinux.org/">world</a>!</p>
EOD
cat <<-EOD | sqlite3 aur.db >actual &&
SELECT RenderedComment FROM PackageComments WHERE ID = 2;
EOD
test_cmp actual expected
'
test_expect_success 'Test HTML sanitizing.' '
cat <<-EOD | sqlite3 aur.db &&
INSERT INTO PackageComments (ID, PackageBaseID, Comments, RenderedComment) VALUES (3, 1, "<script>alert(""XSS!"");</script>", "");
EOD
cover "$RENDERCOMMENT" 3 &&
cat <<-EOD >expected &&
&lt;script&gt;alert("XSS!");&lt;/script&gt;
EOD
cat <<-EOD | sqlite3 aur.db >actual &&
SELECT RenderedComment FROM PackageComments WHERE ID = 3;
EOD
test_cmp actual expected
'
test_expect_success 'Test link conversion.' '
cat <<-EOD | sqlite3 aur.db &&
INSERT INTO PackageComments (ID, PackageBaseID, Comments, RenderedComment) VALUES (4, 1, "
Visit https://www.archlinux.org/#_test_.
Visit *https://www.archlinux.org/*.
Visit <https://www.archlinux.org/>.
Visit \`https://www.archlinux.org/\`.
Visit [Arch Linux](https://www.archlinux.org/).
Visit [Arch Linux][arch].
[arch]: https://www.archlinux.org/
", "");
EOD
cover "$RENDERCOMMENT" 4 &&
cat <<-EOD >expected &&
<p>Visit <a href="https://www.archlinux.org/#_test_">https://www.archlinux.org/#_test_</a>.
Visit <em><a href="https://www.archlinux.org/">https://www.archlinux.org/</a></em>.
Visit <a href="https://www.archlinux.org/">https://www.archlinux.org/</a>.
Visit <code>https://www.archlinux.org/</code>.
Visit <a href="https://www.archlinux.org/">Arch Linux</a>.
Visit <a href="https://www.archlinux.org/">Arch Linux</a>.</p>
EOD
cat <<-EOD | sqlite3 aur.db >actual &&
SELECT RenderedComment FROM PackageComments WHERE ID = 4;
EOD
test_cmp actual expected
'
test_expect_success 'Test Git commit linkification.' '
local oid=`git -C aur.git rev-parse --verify HEAD`
cat <<-EOD | sqlite3 aur.db &&
INSERT INTO PackageComments (ID, PackageBaseID, Comments, RenderedComment) VALUES (5, 1, "
$oid
${oid:0:7}
x.$oid.x
${oid}x
0123456789abcdef
\`$oid\`
http://example.com/$oid
", "");
EOD
cover "$RENDERCOMMENT" 5 &&
cat <<-EOD >expected &&
<p><a href="https://aur.archlinux.org/cgit/aur.git/log/?h=foobar&amp;id=${oid:0:12}">${oid:0:12}</a>
<a href="https://aur.archlinux.org/cgit/aur.git/log/?h=foobar&amp;id=${oid:0:7}">${oid:0:7}</a>
x.<a href="https://aur.archlinux.org/cgit/aur.git/log/?h=foobar&amp;id=${oid:0:12}">${oid:0:12}</a>.x
${oid}x
0123456789abcdef
<code>$oid</code>
<a href="http://example.com/$oid">http://example.com/$oid</a></p>
EOD
cat <<-EOD | sqlite3 aur.db >actual &&
SELECT RenderedComment FROM PackageComments WHERE ID = 5;
EOD
test_cmp actual expected
'
test_expect_success 'Test Flyspray issue linkification.' '
sqlite3 aur.db <<-EOD &&
INSERT INTO PackageComments (ID, PackageBaseID, Comments, RenderedComment) VALUES (6, 1, "
FS#1234567.
*FS#1234*
FS#
XFS#1
\`FS#1234\`
https://archlinux.org/?test=FS#1234
", "");
EOD
cover "$RENDERCOMMENT" 6 &&
cat <<-EOD >expected &&
<p><a href="https://bugs.archlinux.org/task/1234567">FS#1234567</a>.
<em><a href="https://bugs.archlinux.org/task/1234">FS#1234</a></em>
FS#
XFS#1
<code>FS#1234</code>
<a href="https://archlinux.org/?test=FS#1234">https://archlinux.org/?test=FS#1234</a></p>
EOD
sqlite3 aur.db <<-EOD >actual &&
SELECT RenderedComment FROM PackageComments WHERE ID = 6;
EOD
test_cmp actual expected
'
test_expect_success 'Test headings lowering.' '
sqlite3 aur.db <<-EOD &&
INSERT INTO PackageComments (ID, PackageBaseID, Comments, RenderedComment) VALUES (7, 1, "
# One
## Two
### Three
#### Four
##### Five
###### Six
", "");
EOD
cover "$RENDERCOMMENT" 7 &&
cat <<-EOD >expected &&
<h5>One</h5>
<h6>Two</h6>
<h6>Three</h6>
<h6>Four</h6>
<h6>Five</h6>
<h6>Six</h6>
EOD
sqlite3 aur.db <<-EOD >actual &&
SELECT RenderedComment FROM PackageComments WHERE ID = 7;
EOD
test_cmp actual expected
'
test_done

202
test/test_rendercomment.py Normal file
View file

@ -0,0 +1,202 @@
from datetime import datetime
from unittest import mock
import pytest
from aurweb import config, db, logging
from aurweb.models import Package, PackageBase, PackageComment, User
from aurweb.models.account_type import USER_ID
from aurweb.scripts import rendercomment
from aurweb.scripts.rendercomment import update_comment_render
from aurweb.testing.git import GitRepository
logger = logging.get_logger(__name__)
aur_location = config.get("options", "aur_location")
@pytest.fixture(autouse=True)
def setup(db_test, git: GitRepository):
config_get = config.get
def mock_config_get(section: str, key: str) -> str:
if section == "serve" and key == "repo-path":
return git.file_lock.path
elif section == "options" and key == "commit_uri":
return "/cgit/aur.git/log/?h=%s&id=%s"
return config_get(section, key)
with mock.patch("aurweb.config.get", side_effect=mock_config_get):
yield
@pytest.fixture
def user() -> User:
with db.begin():
user = db.create(User, Username="test", Email="test@example.org",
Passwd=str(), AccountTypeID=USER_ID)
yield user
@pytest.fixture
def pkgbase(user: User) -> PackageBase:
now = int(datetime.utcnow().timestamp())
with db.begin():
pkgbase = db.create(PackageBase, Packager=user, Name="pkgbase_0",
SubmittedTS=now, ModifiedTS=now)
yield pkgbase
@pytest.fixture
def package(pkgbase: PackageBase) -> Package:
with db.begin():
package = db.create(Package, PackageBase=pkgbase,
Name=pkgbase.Name, Version="1.0")
yield package
def create_comment(user: User, pkgbase: PackageBase, comments: str,
render: bool = True):
with db.begin():
comment = db.create(PackageComment, User=user,
PackageBase=pkgbase, Comments=comments)
if render:
update_comment_render(comment)
return comment
def test_comment_rendering(user: User, pkgbase: PackageBase):
text = "Hello world! This is a comment."
comment = create_comment(user, pkgbase, text)
expected = f"<p>{text}</p>"
assert comment.RenderedComment == expected
def test_rendercomment_main(user: User, pkgbase: PackageBase):
text = "Hello world! This is a comment."
comment = create_comment(user, pkgbase, text, False)
args = ["aurweb-rendercomment", str(comment.ID)]
with mock.patch("sys.argv", args):
rendercomment.main()
db.refresh(comment)
expected = f"<p>{text}</p>"
assert comment.RenderedComment == expected
def test_markdown_conversion(user: User, pkgbase: PackageBase):
text = "*Hello* [world](https://aur.archlinux.org)!"
comment = create_comment(user, pkgbase, text)
expected = ('<p><em>Hello</em> '
'<a href="https://aur.archlinux.org">world</a>!</p>')
assert comment.RenderedComment == expected
def test_html_sanitization(user: User, pkgbase: PackageBase):
text = '<script>alert("XSS!")</script>'
comment = create_comment(user, pkgbase, text)
expected = '&lt;script&gt;alert("XSS!")&lt;/script&gt;'
assert comment.RenderedComment == expected
def test_link_conversion(user: User, pkgbase: PackageBase):
text = """\
Visit https://www.archlinux.org/#_test_.
Visit *https://www.archlinux.org/*.
Visit <https://www.archlinux.org/>.
Visit `https://www.archlinux.org/`.
Visit [Arch Linux](https://www.archlinux.org/).
Visit [Arch Linux][arch].
[arch]: https://www.archlinux.org/\
"""
comment = create_comment(user, pkgbase, text)
expected = '''\
<p>Visit <a href="https://www.archlinux.org/#_test_">\
https://www.archlinux.org/#_test_</a>.
Visit <em><a href="https://www.archlinux.org/">https://www.archlinux.org/</a></em>.
Visit <a href="https://www.archlinux.org/">https://www.archlinux.org/</a>.
Visit <code>https://www.archlinux.org/</code>.
Visit <a href="https://www.archlinux.org/">Arch Linux</a>.
Visit <a href="https://www.archlinux.org/">Arch Linux</a>.</p>\
'''
assert comment.RenderedComment == expected
def test_git_commit_link(git: GitRepository, user: User, package: Package):
commit_hash = git.commit(package, "Initial commit.")
logger.info(f"Created commit: {commit_hash}")
logger.info(f"Short hash: {commit_hash[:7]}")
text = f"""\
{commit_hash}
{commit_hash[:7]}
x.{commit_hash}.x
{commit_hash}x
0123456789abcdef
`{commit_hash}`
http://example.com/{commit_hash}\
"""
comment = create_comment(user, package.PackageBase, text)
pkgname = package.PackageBase.Name
cgit_path = f"/cgit/aur.git/log/?h={pkgname}&amp;"
expected = f"""\
<p><a href="{cgit_path}id={commit_hash[:12]}">{commit_hash[:12]}</a>
<a href="{cgit_path}id={commit_hash[:7]}">{commit_hash[:7]}</a>
x.<a href="{cgit_path}id={commit_hash[:12]}">{commit_hash[:12]}</a>.x
{commit_hash}x
0123456789abcdef
<code>{commit_hash}</code>
<a href="http://example.com/{commit_hash}">\
http://example.com/{commit_hash}\
</a>\
</p>\
"""
assert comment.RenderedComment == expected
def test_flyspray_issue_link(user: User, pkgbase: PackageBase):
text = """\
FS#1234567.
*FS#1234*
FS#
XFS#1
`FS#1234`
https://archlinux.org/?test=FS#1234\
"""
comment = create_comment(user, pkgbase, text)
expected = """\
<p><a href="https://bugs.archlinux.org/task/1234567">FS#1234567</a>.
<em><a href="https://bugs.archlinux.org/task/1234">FS#1234</a></em>
FS#
XFS#1
<code>FS#1234</code>
<a href="https://archlinux.org/?test=FS#1234">\
https://archlinux.org/?test=FS#1234\
</a>\
</p>\
"""
assert comment.RenderedComment == expected
def test_lower_headings(user: User, pkgbase: PackageBase):
text = """\
# One
## Two
### Three
#### Four
##### Five
###### Six\
"""
comment = create_comment(user, pkgbase, text)
expected = """\
<h5>One</h5>
<h6>Two</h6>
<h6>Three</h6>
<h6>Four</h6>
<h6>Five</h6>
<h6>Six</h6>\
"""
assert comment.RenderedComment == expected

17
test/test_util.py
View file

@ -40,3 +40,20 @@ def test_round():
assert filters.do_round(1.3) == 1
assert filters.do_round(1.5) == 2
assert filters.do_round(2.0) == 2
def test_git_search():
""" Test that git_search matches the full commit if necessary. """
commit_hash = "0123456789abcdef"
repo = {commit_hash}
prefixlen = util.git_search(repo, commit_hash)
assert prefixlen == 16
def test_git_search_double_commit():
""" Test that git_search matches a shorter prefix length. """
commit_hash = "0123456789abcdef"
repo = {commit_hash[:13]}
# Locate the shortest prefix length that matches commit_hash.
prefixlen = util.git_search(repo, commit_hash)
assert prefixlen == 13