WIP: Initial message schema for Anitya

.coveragerc

@@ -2,6 +2,8 @@
 branch = True
 include =
     anitya/*
+    anitya_schema/anitya_schema/*
+
 
 [report]
 fail_under = 88
@@ -13,3 +15,4 @@ omit =
     anitya/tests/*
     anitya/db/migrations
     anitya/db/migrations/*
+    anitya_schema/anitya_schema/tests/*

anitya/admin.py

@@ -45,7 +45,7 @@ def edit_distro(distro_name):
         if name != distro.name:
             utilities.log(
                 Session,
-                distro=distro,
+                distro=distro.__json__(),
                 topic='distro.edit',
                 message=dict(
                     agent=flask.g.user.username,
@@ -89,7 +89,7 @@ def delete_distro(distro_name):
     if form.validate_on_submit():
         utilities.log(
             Session,
-            distro=distro,
+            distro=distro.__json__(),
             topic='distro.remove',
             message=dict(
                 agent=flask.g.user.username,
@@ -129,7 +129,7 @@ def delete_project(project_id):
         if confirm:
             utilities.log(
                 Session,
-                project=project,
+                project=project.__json__(),
                 topic='project.remove',
                 message=dict(
                     agent=flask.g.user.username,
@@ -184,7 +184,7 @@ def delete_project_mapping(project_id, distro_name, pkg_name):
         if confirm:
             utilities.log(
                 Session,
-                project=project,
+                project=project.__json__(),
                 topic='project.map.remove',
                 message=dict(
                     agent=flask.g.user.username,
@@ -239,7 +239,7 @@ def delete_project_version(project_id, version):
         if confirm:
             utilities.log(
                 Session,
-                project=project,
+                project=project.__json__(),
                 topic='project.version.remove',
                 message=dict(
                     agent=flask.g.user.username,

anitya/config.py

@@ -97,6 +97,7 @@
                   r'(?:minsrc|src|source|asc|release))?\.(?:tar|t[bglx]z|tbz2|zip)',
     # Token for GitHub API
     GITHUB_ACCESS_TOKEN=None,
+    LEGACY_MESSAGING=False,  # If True, publish with fedmsg instead of fedora_messaging
 )
 
 # Start with a basic logging configuration, which will be replaced by any user-

anitya/lib/utilities.py

@@ -22,6 +22,7 @@
 import logging
 
 import sqlalchemy as sa
+from fedora_messaging import api, message, exceptions as fm_exceptions
 from sqlalchemy import create_engine
 from sqlalchemy.exc import SQLAlchemyError, IntegrityError
 from sqlalchemy.orm import sessionmaker
@@ -30,6 +31,7 @@
 import arrow
 
 from . import plugins, exceptions
+from anitya import config
 from anitya.db import models, Base
 
 
@@ -38,19 +40,29 @@
 
 def fedmsg_publish(*args, **kwargs):  # pragma: no cover
     ''' Try to publish a message on the fedmsg bus. '''
-    # We catch Exception if we want :-p
-    # pylint: disable=W0703
-    # Ignore message about fedmsg import
-    # pylint: disable=F0401
-    kwargs['modname'] = 'anitya'
-    kwargs['cert_prefix'] = 'anitya'
-    kwargs['name'] = 'relay_inbound'
-    kwargs['active'] = True
-    try:
-        import fedmsg
-        fedmsg.publish(*args, **kwargs)
-    except Exception as err:
-        _log.error(str(err))
+    if config.config["LEGACY_MESSAGING"]:
+        # We catch Exception if we want :-p
+        # pylint: disable=W0703
+        # Ignore message about fedmsg import
+        # pylint: disable=F0401
+        kwargs['modname'] = 'anitya'
+        kwargs['cert_prefix'] = 'anitya'
+        kwargs['name'] = 'relay_inbound'
+        kwargs['active'] = True
+        try:
+            import fedmsg
+            fedmsg.publish(*args, **kwargs)
+        except Exception as err:
+            _log.error(str(err))
+    else:
+        try:
+            message_class = message.get_class("anitya." + kwargs["topic"])
+            api.publish(message_class(body=kwargs["msg"]))
+        except (fm_exceptions.ConnectionException, fm_exceptions.PublishException) as err:
+            # For now, continue just logging the error. Once the messaging has
+            # been untangled into SQLAlchemy events, it should probably result
+            # in an exception and the client should try again later.
+            _log.error(str(err))
 
 
 def check_project_release(project, session, test=False):
@@ -133,7 +145,7 @@ def check_project_release(project, session, test=False):
     if publish:
         log(
             session,
-            project=project,
+            project=project.__json__(),
             topic="project.version.update",
             message=dict(
                 project=project.__json__(),
@@ -290,7 +302,7 @@ def create_project(
 
     log(
         session,
-        project=project,
+        project=project.__json__(),
         topic='project.add',
         message=dict(
             agent=user_id,
@@ -352,7 +364,7 @@ def edit_project(
         if changes:
             log(
                 session,
-                project=project,
+                project=project.__json__(),
                 topic='project.edit',
                 message=dict(
                     agent=user_id,
@@ -409,7 +421,7 @@ def map_project(
 
         log(
             session,
-            distro=distro_obj,
+            distro=distro_obj.__json__(),
             topic='distro.add',
             message=dict(
                 agent=user_id,
@@ -488,8 +500,8 @@ def map_project(
 
     log(
         session,
-        project=project,
-        distro=distro_obj,
+        project=project.__json__(),
+        distro=distro_obj.__json__(),
         topic=topic,
         message=message,
     )
@@ -519,7 +531,7 @@ def flag_project(session, project, reason, user_email, user_id):
 
     log(
         session,
-        project=project,
+        project=project.__json__(),
         topic='project.flag',
         message=dict(
             agent=user_id,

anitya/tests/base.py

@@ -23,10 +23,13 @@
 from __future__ import print_function
 
 from contextlib import contextmanager
+import collections
 import unittest
 import os
+import mock
 
 from flask import request_started
+from fedora_messaging import message
 from social_flask_sqlalchemy.models import PSABase
 from sqlalchemy import create_engine, event
 import flask_login
@@ -177,68 +180,66 @@ def create_distro(session):
 
 def create_project(session):
     """ Create some basic projects to work with. """
-    utilities.create_project(
-        session,
+    project = models.Project(
         name='geany',
         homepage='https://www.geany.org/',
+        backend='custom',
         version_url='https://www.geany.org/Download/Releases',
         regex='DEFAULT',
-        user_id='[email protected]',
     )
+    session.add(project)
 
-    utilities.create_project(
-        session,
+    project = models.Project(
         name='subsurface',
         homepage='https://subsurface-divelog.org/',
+        backend='custom',
         version_url='https://subsurface-divelog.org/downloads/',
         regex='DEFAULT',
-        user_id='[email protected]',
     )
+    session.add(project)
 
-    utilities.create_project(
-        session,
+    project = models.Project(
         name='R2spec',
         homepage='https://fedorahosted.org/r2spec/',
-        user_id='[email protected]',
+        backend='custom',
     )
+    session.add(project)
+    session.commit()
 
 
 def create_ecosystem_projects(session):
     """ Create some fake projects from particular upstream ecosystems
 
     Each project name is used in two different ecosystems
     """
-    utilities.create_project(
-        session,
+    project = models.Project(
         name='pypi_and_npm',
         homepage='https://example.com/not-a-real-pypi-project',
         backend='PyPI',
-        user_id='[email protected]'
     )
+    session.add(project)
 
-    utilities.create_project(
-        session,
+    project = models.Project(
         name='pypi_and_npm',
         homepage='https://example.com/not-a-real-npmjs-project',
         backend='npmjs',
-        user_id='[email protected]'
     )
+    session.add(project)
 
-    utilities.create_project(
-        session,
+    project = models.Project(
         name='rubygems_and_maven',
         homepage='https://example.com/not-a-real-rubygems-project',
         backend='Rubygems',
-        user_id='[email protected]'
     )
+    session.add(project)
 
-    utilities.create_project(
-        session,
+    project = models.Project(
         name='rubygems_and_maven',
         homepage='https://example.com/not-a-real-maven-project',
         backend='Maven Central',
-        user_id='[email protected]'
     )
+    session.add(project)
+    session.commit()
 
 
 def create_package(session):
@@ -262,25 +263,19 @@ def create_package(session):
 
 def create_flagged_project(session):
     """ Create and flag a project. Returns the ProjectFlag. """
-    project = utilities.create_project(
-        session,
+    project = models.Project(
         name='geany',
         homepage='https://www.geany.org/',
         version_url='https://www.geany.org/Download/Releases',
         regex='DEFAULT',
-        user_id='[email protected]',
     )
-
     session.add(project)
 
-    flag = utilities.flag_project(
-        session,
-        project,
-        "This is a duplicate.",
-        "[email protected]",
-        "user_openid_id",
+    flag = models.ProjectFlag(
+        project=project,
+        reason="this is a duplicate.",
+        user="[email protected]",
     )
-
     session.add(flag)
 
     session.commit()

anitya/tests/test_config.py

@@ -54,6 +54,7 @@
 
 default_regex = "a*b*"
 github_access_token = "foobar"
+legacy_messaging = true
 
 [anitya_log_config]
     version = 1
@@ -171,6 +172,7 @@ def test_full_config_file(self, mock_exists, mock_log):
             'LIBRARIESIO_PLATFORM_WHITELIST': ['pypi', 'rubygems'],
             'DEFAULT_REGEX': 'a*b*',
             'GITHUB_ACCESS_TOKEN': 'foobar',
+            'LEGACY_MESSAGING': True,
         }
         config = anitya_config.load()
         self.assertEqual(sorted(expected_config.keys()), sorted(config.keys()))