aurweb: Globalize a Translator instance, add more utility

+ Added SUPPORTED_LANGUAGES, a global constant dictionary of
  language => display pairs for languages we support.
+ Add Translator.get_translator, a function used to retrieve a
  translator after initializing it (if needed). Use `fallback=True`
  while creating languages, in case we setup a language that we
  don't have a translation for, it will noop the translation.
  This is particularly useful for "en," since we do not translate
  it, but doing this will allow us to go through our normal translation
  flow in any case.
+ Added typing.
+ Added get_request_language, a function that grabs the language for
  a request session, defaulting to aurweb.config [options] default_lang.
+ Added get_raw_translator_for_request, a function that retrieves
  the concrete translation object for a given language.
+ Added tr, a jinja2 contextfilter that can be used to inline translate
  strings in jinja2 templates.
+ Added `python-jinja` dep to .gitlab-ci.yml. This needs to be
  included in documentation before this set is merged in.
+ Introduce pytest units (test_l10n.py) in `test` along with
  __init__.py, which marks `test` as a test package.
+ Additionally, fix up notify.py to use the global translator. Also
  reduce its source width to <= 80 by newlining some code.
+ Additionally, prepare locale in .gitlab-ci.yml and add
  aurweb.config [options] localedir to config.dev with YOUR_AUR_ROOT
  like others.

Signed-off-by: Kevin Morris <kevr@0cost.org>
Signed-off-by: Lukas Fleischer <lfleischer@archlinux.org>

.gitlab-ci.yml

@@ -20,6 +20,7 @@ test:
   script:
     - python setup.py install
     - sed -r "s;YOUR_AUR_ROOT;$(pwd);g" conf/config.dev > conf/config
+    - AUR_CONFIG=conf/config make -C po all install
     - AUR_CONFIG=conf/config python -m aurweb.initdb
     - make -C test
     - coverage report --include='aurweb/*'

aurweb/l10n.py

@@ -1,24 +1,79 @@
 import gettext
+import typing
+
+from collections import OrderedDict
+
+from fastapi import Request
+from jinja2 import contextfilter
 
 import aurweb.config
 
+SUPPORTED_LANGUAGES = OrderedDict({
+    "ar": "العربية",
+    "ast": "Asturianu",
+    "ca": "Català",
+    "cs": "Český",
+    "da": "Dansk",
+    "de": "Deutsch",
+    "el": "Ελληνικά",
+    "en": "English",
+    "es": "Español",
+    "es_419": "Español (Latinoamérica)",
+    "fi": "Suomi",
+    "fr": "Français",
+    "he": "עברית",
+    "hr": "Hrvatski",
+    "hu": "Magyar",
+    "it": "Italiano",
+    "ja": "日本語",
+    "nb": "Norsk",
+    "nl": "Nederlands",
+    "pl": "Polski",
+    "pt_BR": "Português (Brasil)",
+    "pt_PT": "Português (Portugal)",
+    "ro": "Română",
+    "ru": "Русский",
+    "sk": "Slovenčina",
+    "sr": "Srpski",
+    "tr": "Türkçe",
+    "uk": "Українська",
+    "zh_CN": "简体中文",
+    "zh_TW": "正體中文"
+})
+
 
 class Translator:
     def __init__(self):
         self._localedir = aurweb.config.get('options', 'localedir')
         self._translator = {}
 
-    def translate(self, s, lang):
+    def get_translator(self, lang: str):
-        if lang == 'en':
-            return s
         if lang not in self._translator:
             self._translator[lang] = gettext.translation("aurweb",
                                                          self._localedir,
-                                                         languages=[lang])
+                                                         languages=[lang],
-        return self._translator[lang].gettext(s)
+                                                         fallback=True)
+        return self._translator.get(lang)
+
+    def translate(self, s: str, lang: str):
+        return self.get_translator(lang).gettext(s)
 
 
-def get_translator_for_request(request):
+# Global translator object.
+translator = Translator()
+
+
+def get_request_language(request: Request):
+    return request.cookies.get("AURLANG",
+                               aurweb.config.get("options", "default_lang"))
+
+
+def get_raw_translator_for_request(request: Request):
+    lang = get_request_language(request)
+    return translator.get_translator(lang)
+
+
+def get_translator_for_request(request: Request):
     """
     Determine the preferred language from a FastAPI request object and build a
     translator function for it.
@@ -29,12 +84,16 @@ def get_translator_for_request(request):
     print(_("Hello"))
     ```
     """
-    lang = request.cookies.get("AURLANG")
+    lang = get_request_language(request)
-    if lang is None:
-        lang = aurweb.config.get("options", "default_lang")
-    translator = Translator()
 
     def translate(message):
         return translator.translate(message, lang)
 
     return translate
+
+
+@contextfilter
+def tr(context: typing.Any, value: str):
+    """ A translation filter; example: {{ "Hello" | tr("de") }}. """
+    _ = get_translator_for_request(context.get("request"))
+    return _(value)

aurweb/scripts/notify.py

@@ -40,9 +40,6 @@ def pkgbase_from_pkgreq(conn, reqid):
 
 
 class Notification:
-    def __init__(self):
-        self._l10n = aurweb.l10n.Translator()
-
     def get_refs(self):
         return ()
 
@@ -97,9 +94,12 @@ class Notification:
             else:
                 # send email using smtplib; no local MTA required
                 server_addr = aurweb.config.get('notifications', 'smtp-server')
-                server_port = aurweb.config.getint('notifications', 'smtp-port')
+                server_port = aurweb.config.getint('notifications',
-                use_ssl = aurweb.config.getboolean('notifications', 'smtp-use-ssl')
+                                                   'smtp-port')
-                use_starttls = aurweb.config.getboolean('notifications', 'smtp-use-starttls')
+                use_ssl = aurweb.config.getboolean('notifications',
+                                                   'smtp-use-ssl')
+                use_starttls = aurweb.config.getboolean('notifications',
+                                                        'smtp-use-starttls')
                 user = aurweb.config.get('notifications', 'smtp-user')
                 passwd = aurweb.config.get('notifications', 'smtp-password')
 
@@ -127,7 +127,8 @@ class ResetKeyNotification(Notification):
         cur = conn.execute('SELECT UserName, Email, BackupEmail, ' +
                            'LangPreference, ResetKey ' +
                            'FROM Users WHERE ID = ? AND Suspended = 0', [uid])
-        self._username, self._to, self._backup, self._lang, self._resetkey = cur.fetchone()
+        self._username, self._to, self._backup, self._lang, self._resetkey = \
+            cur.fetchone()
         super().__init__()
 
     def get_recipients(self):
@@ -137,10 +138,10 @@ class ResetKeyNotification(Notification):
             return [(self._to, self._lang)]
 
     def get_subject(self, lang):
-        return self._l10n.translate('AUR Password Reset', lang)
+        return aurweb.l10n.translator.translate('AUR Password Reset', lang)
 
     def get_body(self, lang):
-        return self._l10n.translate(
+        return aurweb.l10n.translator.translate(
             'A password reset request was submitted for the account '
             '{user} associated with your email address. If you wish to '
             'reset your password follow the link [1] below, otherwise '
@@ -153,11 +154,12 @@ class ResetKeyNotification(Notification):
 
 class WelcomeNotification(ResetKeyNotification):
     def get_subject(self, lang):
-        return self._l10n.translate('Welcome to the Arch User Repository',
+        return aurweb.l10n.translator.translate(
+            'Welcome to the Arch User Repository',
             lang)
 
     def get_body(self, lang):
-        return self._l10n.translate(
+        return aurweb.l10n.translator.translate(
             'Welcome to the Arch User Repository! In order to set an '
             'initial password for your new account, please click the '
             'link [1] below. If the link does not work, try copying and '
@@ -186,16 +188,18 @@ class CommentNotification(Notification):
         return self._recipients
 
     def get_subject(self, lang):
-        return self._l10n.translate('AUR Comment for {pkgbase}',
+        return aurweb.l10n.translator.translate(
+            'AUR Comment for {pkgbase}',
             lang).format(pkgbase=self._pkgbase)
 
     def get_body(self, lang):
-        body = self._l10n.translate(
+        body = aurweb.l10n.translator.translate(
             '{user} [1] added the following comment to {pkgbase} [2]:',
             lang).format(user=self._user, pkgbase=self._pkgbase)
         body += '\n\n' + self._text + '\n\n-- \n'
-        dnlabel = self._l10n.translate('Disable notifications', lang)
+        dnlabel = aurweb.l10n.translator.translate(
-        body += self._l10n.translate(
+            'Disable notifications', lang)
+        body += aurweb.l10n.translator.translate(
             'If you no longer wish to receive notifications about this '
             'package, please go to the package page [2] and select '
             '"{label}".', lang).format(label=dnlabel)
@@ -231,17 +235,18 @@ class UpdateNotification(Notification):
         return self._recipients
 
     def get_subject(self, lang):
-        return self._l10n.translate('AUR Package Update: {pkgbase}',
+        return aurweb.l10n.translator.translate(
+            'AUR Package Update: {pkgbase}',
             lang).format(pkgbase=self._pkgbase)
 
     def get_body(self, lang):
-        body = self._l10n.translate('{user} [1] pushed a new commit to '
+        body = aurweb.l10n.translator.translate(
-                                    '{pkgbase} [2].', lang).format(
+            '{user} [1] pushed a new commit to {pkgbase} [2].',
-                                            user=self._user,
+            lang).format(user=self._user, pkgbase=self._pkgbase)
-                                            pkgbase=self._pkgbase)
         body += '\n\n-- \n'
-        dnlabel = self._l10n.translate('Disable notifications', lang)
+        dnlabel = aurweb.l10n.translator.translate(
-        body += self._l10n.translate(
+            'Disable notifications', lang)
+        body += aurweb.l10n.translator.translate(
             'If you no longer wish to receive notifications about this '
             'package, please go to the package page [2] and select '
             '"{label}".', lang).format(label=dnlabel)
@@ -261,7 +266,8 @@ class FlagNotification(Notification):
     def __init__(self, conn, uid, pkgbase_id):
         self._user = username_from_id(conn, uid)
         self._pkgbase = pkgbase_from_id(conn, pkgbase_id)
-        cur = conn.execute('SELECT DISTINCT Users.Email, ' +
+        cur = conn.execute(
+            'SELECT DISTINCT Users.Email, ' +
             'Users.LangPreference FROM Users ' +
             'LEFT JOIN PackageComaintainers ' +
             'ON PackageComaintainers.UsersID = Users.ID ' +
@@ -280,12 +286,12 @@ class FlagNotification(Notification):
         return self._recipients
 
     def get_subject(self, lang):
-        return self._l10n.translate('AUR Out-of-date Notification for '
+        return aurweb.l10n.translator.translate(
-                                    '{pkgbase}',
+            'AUR Out-of-date Notification for {pkgbase}',
             lang).format(pkgbase=self._pkgbase)
 
     def get_body(self, lang):
-        body = self._l10n.translate(
+        body = aurweb.l10n.translator.translate(
             'Your package {pkgbase} [1] has been flagged out-of-date by '
             '{user} [2]:', lang).format(pkgbase=self._pkgbase,
                                         user=self._user)
@@ -320,7 +326,8 @@ class OwnershipEventNotification(Notification):
         return self._recipients
 
     def get_subject(self, lang):
-        return self._l10n.translate('AUR Ownership Notification for {pkgbase}',
+        return aurweb.l10n.translator.translate(
+            'AUR Ownership Notification for {pkgbase}',
             lang).format(pkgbase=self._pkgbase)
 
     def get_refs(self):
@@ -330,14 +337,14 @@ class OwnershipEventNotification(Notification):
 
 class AdoptNotification(OwnershipEventNotification):
     def get_body(self, lang):
-        return self._l10n.translate(
+        return aurweb.l10n.translator.translate(
             'The package {pkgbase} [1] was adopted by {user} [2].',
             lang).format(pkgbase=self._pkgbase, user=self._user)
 
 
 class DisownNotification(OwnershipEventNotification):
     def get_body(self, lang):
-        return self._l10n.translate(
+        return aurweb.l10n.translator.translate(
             'The package {pkgbase} [1] was disowned by {user} '
             '[2].', lang).format(pkgbase=self._pkgbase,
                                  user=self._user)
@@ -355,8 +362,8 @@ class ComaintainershipEventNotification(Notification):
         return [(self._to, self._lang)]
 
     def get_subject(self, lang):
-        return self._l10n.translate('AUR Co-Maintainer Notification for '
+        return aurweb.l10n.translator.translate(
-                                    '{pkgbase}',
+            'AUR Co-Maintainer Notification for {pkgbase}',
             lang).format(pkgbase=self._pkgbase)
 
     def get_refs(self):
@@ -365,14 +372,14 @@ class ComaintainershipEventNotification(Notification):
 
 class ComaintainerAddNotification(ComaintainershipEventNotification):
     def get_body(self, lang):
-        return self._l10n.translate(
+        return aurweb.l10n.translator.translate(
             'You were added to the co-maintainer list of {pkgbase} [1].',
             lang).format(pkgbase=self._pkgbase)
 
 
 class ComaintainerRemoveNotification(ComaintainershipEventNotification):
     def get_body(self, lang):
-        return self._l10n.translate(
+        return aurweb.l10n.translator.translate(
             'You were removed from the co-maintainer list of {pkgbase} '
             '[1].', lang).format(pkgbase=self._pkgbase)
 
@@ -400,13 +407,15 @@ class DeleteNotification(Notification):
         return self._recipients
 
     def get_subject(self, lang):
-        return self._l10n.translate('AUR Package deleted: {pkgbase}',
+        return aurweb.l10n.translator.translate(
+            'AUR Package deleted: {pkgbase}',
             lang).format(pkgbase=self._old_pkgbase)
 
     def get_body(self, lang):
         if self._new_pkgbase:
-            dnlabel = self._l10n.translate('Disable notifications', lang)
+            dnlabel = aurweb.l10n.translator.translate(
-            return self._l10n.translate(
+                'Disable notifications', lang)
+            return aurweb.l10n.translator.translate(
                 '{user} [1] merged {old} [2] into {new} [3].\n\n'
                 '-- \n'
                 'If you no longer wish receive notifications about the '
@@ -414,7 +423,7 @@ class DeleteNotification(Notification):
                 lang).format(user=self._user, old=self._old_pkgbase,
                              new=self._new_pkgbase, label=dnlabel)
         else:
-            return self._l10n.translate(
+            return aurweb.l10n.translator.translate(
                 '{user} [1] deleted {pkgbase} [2].\n\n'
                 'You will no longer receive notifications about this '
                 'package.', lang).format(user=self._user,
@@ -432,7 +441,8 @@ class RequestOpenNotification(Notification):
     def __init__(self, conn, uid, reqid, reqtype, pkgbase_id, merge_into=None):
         self._user = username_from_id(conn, uid)
         self._pkgbase = pkgbase_from_id(conn, pkgbase_id)
-        cur = conn.execute('SELECT DISTINCT Users.Email FROM PackageRequests ' +
+        cur = conn.execute(
+            'SELECT DISTINCT Users.Email FROM PackageRequests ' +
             'INNER JOIN PackageBases ' +
             'ON PackageBases.ID = PackageRequests.PackageBaseID ' +
             'INNER JOIN Users ' +
@@ -489,7 +499,8 @@ class RequestOpenNotification(Notification):
 class RequestCloseNotification(Notification):
     def __init__(self, conn, uid, reqid, reason):
         self._user = username_from_id(conn, uid) if int(uid) else None
-        cur = conn.execute('SELECT DISTINCT Users.Email FROM PackageRequests ' +
+        cur = conn.execute(
+            'SELECT DISTINCT Users.Email FROM PackageRequests ' +
             'INNER JOIN PackageBases ' +
             'ON PackageBases.ID = PackageRequests.PackageBaseID ' +
             'INNER JOIN Users ' +
@@ -563,11 +574,12 @@ class TUVoteReminderNotification(Notification):
         return self._recipients
 
     def get_subject(self, lang):
-        return self._l10n.translate('TU Vote Reminder: Proposal {id}',
+        return aurweb.l10n.translator.translate(
+            'TU Vote Reminder: Proposal {id}',
             lang).format(id=self._vote_id)
 
     def get_body(self, lang):
-        return self._l10n.translate(
+        return aurweb.l10n.translator.translate(
             'Please remember to cast your vote on proposal {id} [1]. '
             'The voting period ends in less than 48 hours.',
             lang).format(id=self._vote_id)

conf/config.dev

@@ -19,6 +19,7 @@ name = YOUR_AUR_ROOT/aurweb.sqlite3
 aur_location = http://127.0.0.1:8080
 disable_http_login = 0
 enable-maintenance = 0
+localedir = YOUR_AUR_ROOT/web/locale
 
 ; Single sign-on; see doc/sso.txt.
 [sso]

test/test_l10n.py

@@ -0,0 +1,44 @@
+""" Test our l10n module. """
+from aurweb import l10n
+
+
+class FakeRequest:
+    """ A fake Request doppleganger; use this to change request.cookies
+    easily and with no side-effects. """
+
+    def __init__(self, *args, **kwargs):
+        self.cookies = kwargs.pop("cookies", dict())
+
+
+def test_translator():
+    """ Test creating l10n translation tools. """
+    de_home = l10n.translator.translate("Home", "de")
+    assert de_home == "Startseite"
+
+
+def test_get_request_language():
+    """ First, tests default_lang, then tests a modified AURLANG cookie. """
+    request = FakeRequest()
+    assert l10n.get_request_language(request) == "en"
+
+    request.cookies["AURLANG"] = "de"
+    assert l10n.get_request_language(request) == "de"
+
+
+def test_get_raw_translator_for_request():
+    """ Make sure that get_raw_translator_for_request is giving us
+    the translator we expect. """
+    request = FakeRequest(cookies={"AURLANG": "de"})
+
+    translator = l10n.get_raw_translator_for_request(request)
+    assert translator.gettext("Home") == \
+        l10n.translator.translate("Home", "de")
+
+
+def test_get_translator_for_request():
+    """ Make sure that get_translator_for_request is giving us back
+    our expected translation function. """
+    request = FakeRequest(cookies={"AURLANG": "de"})
+
+    translate = l10n.get_translator_for_request(request)
+    assert translate("Home") == "Startseite"