Actually store latest known version cursor

anitya/db/migrations/versions/136d119ab015_rename_project_latest_known_cursor_to_.py

@@ -0,0 +1,27 @@
+"""rename Project.latest_known_cursor to latest_version_cursor
+
+Revision ID: 136d119ab015
+Revises: 92a2e9eba0a7
+Create Date: 2019-12-20 20:56:25.783460
+"""
+
+from alembic import op
+
+
+# revision identifiers, used by Alembic.
+revision = "136d119ab015"
+down_revision = "92a2e9eba0a7"
+
+
+def upgrade():
+    with op.batch_alter_table("projects") as batch_op:
+        batch_op.alter_column(
+            "latest_known_cursor", new_column_name="latest_version_cursor"
+        )
+
+
+def downgrade():
+    with op.batch_alter_table("projects") as batch_op:
+        batch_op.alter_column(
+            "latest_version_cursor", new_column_name="latest_known_cursor"
+        )

anitya/db/migrations/versions/92a2e9eba0a7_add_project_latest_known_cursor.py

@@ -22,4 +22,5 @@ def upgrade():
 
 
 def downgrade():
-    op.drop_column("projects", "latest_known_cursor")
+    with op.batch_alter_table("projects") as batch_op:
+        batch_op.drop_column("latest_known_cursor")

anitya/db/models.py

@@ -213,8 +213,12 @@ class Project(Base):
             offered by the server at ``version_url``. Defaults to ``False``.
         releases_only (sa.Boolean): Whether or not to check releases instead of tags.
             This is now only used by GitHub backend.
-        latest_version (sa.Boolean): The latest version for the project, as determined
+        latest_version (sa.String): The latest version for the project, as determined
             by the version sorting algorithm.
+        latest_version_cursor (sa.String): A backend-specific cursor to the
+            latest version for the project, as determined by the version
+            sorting algorithm. This can be used to avoid paginating over every
+            old version.
         logs (sa.Text): The result of the last update.
         check_successful (sa.Boolean): Flag that contains result of last check.
             ``None`` - not checked yet, ``True`` - checked successfully, ``False``
@@ -245,7 +249,7 @@ class Project(Base):
     version_scheme = sa.Column(sa.String(50), nullable=True)
 
     latest_version = sa.Column(sa.String(50))
-    latest_known_cursor = sa.Column(sa.String(200), nullable=True)
+    latest_version_cursor = sa.Column(sa.String(200), nullable=True)
     logs = sa.Column(sa.Text)
     check_successful = sa.Column(sa.Boolean, default=None, index=True)
 

anitya/lib/backends/github.py

@@ -132,11 +132,11 @@ def get_versions(cls, project):
                 )
             )
 
-        if project.latest_known_cursor:
+        if project.latest_version_cursor:
             # If we know about the cursor of the latest version, attempt to
             # limit results to anything after it. Only if that fails, try
             # without one.
-            cursor_attempts = (project.latest_known_cursor, None)
+            cursor_attempts = (project.latest_version_cursor, None)
         else:
             cursor_attempts = (None,)
 
@@ -185,7 +185,7 @@ def get_versions(cls, project):
                 )
             ):
                 # unset faulty cursor for now
-                project.latest_known_cursor = None
+                project.latest_version_cursor = None
                 continue
 
             versions = parse_json(json, project)
@@ -259,7 +259,8 @@ def parse_json(json, project):
     versions = []
 
     for edge in json_data["edges"]:
-        version = {"cursor": edge.get("cursor")}
+        version = {"cursor": edge["cursor"]}
+
         if project.releases_only:
             version["version"] = edge["node"]["tag"]["name"]
         else:

anitya/lib/utilities.py

@@ -141,11 +141,14 @@ def check_project_release(project, session, test=False):
 
     sorted_versions = project.get_sorted_version_objects()
     if sorted_versions:
-        max_version = sorted_versions[0].parse()
-        max_version_cursor = sorted_versions[0].cursor
+        max_version_obj = sorted_versions[0]
+        max_version = max_version_obj.parse()
     if project.latest_version != max_version:
         project.latest_version = max_version
-        project.latest_known_cursor = max_version_cursor
+        if versions:  # pragma: no branch
+            # project.create_version_objects() returns sorted versions, i.e.
+            # version[-1] will be the latest one
+            project.latest_version_cursor = versions[-1].cursor
         publish = True
     else:
         project.logs = "No new version found"

anitya/tests/lib/backends/test_github.py

@@ -342,7 +342,7 @@ def test_get_versions_valid_with_valid_cursor(self):
         project = self.projects["valid_with_version_url"]
         all_versions = self.expected_versions["valid_with_version_url"]
         lk_version, lk_cursor = self.version_with_cursor["valid_with_version_url"]
-        project.latest_known_cursor = lk_cursor
+        project.latest_version_cursor = lk_cursor
         exp = all_versions[all_versions.index(lk_version) + 1 :]
         obs = backend.GithubBackend.get_ordered_versions(project)
         self.assertEqual(obs, exp)
@@ -353,7 +353,7 @@ def test_get_versions_valid_with_invalid_cursor(self):
 
         It should return all versions."""
         project = self.projects["valid_with_version_url"]
-        project.latest_known_cursor = "invalid cursor"
+        project.latest_version_cursor = "invalid cursor"
         exp = self.expected_versions["valid_with_version_url"]
         obs = backend.GithubBackend.get_ordered_versions(project)
         self.assertEqual(obs, exp)
@@ -464,6 +464,12 @@ class JsonTests(unittest.TestCase):
     def __init__(self, *args, **kwargs):
         super(JsonTests, self).__init__(*args, **kwargs)
         self.maxDiff = None
+        self.project = models.Project(
+            name="foobar",
+            homepage="https://foobar.com",
+            version_url="foo/bar",
+            backend=BACKEND,
+        )
 
     def test_prepare_query_after(self):
         """ Assert query creation with cursor """
@@ -541,17 +547,33 @@ def test_prepare_query_releases(self):
         self.assertMultiLineEqual(exp, obs)
 
     def test_parse_json(self):
-        """ Assert parsing json"""
-        project = models.Project(
-            name="foobar",
-            homepage="https://foobar.com",
-            version_url="foo/bar",
-            backend=BACKEND,
-        )
+        """Test parsing a JSON response without errors."""
+        json = {
+            "data": {
+                "repository": {
+                    "refs": {
+                        "totalCount": 1,
+                        "edges": [{"cursor": "cUrSoR", "node": {"name": "1.0"}}],
+                    },
+                },
+                "rateLimit": {"limit": 5000, "remaining": 5000, "resetAt": "dummy"},
+            }
+        }
+        exp = [{"cursor": "cUrSoR", "version": "1.0"}]
+        obs = backend.parse_json(json, self.project)
+        self.assertEqual(exp, obs)
+
+    def test_parse_json_with_errors(self):
+        """Test parsing JSON flagging errors."""
         json = {"errors": [{"type": "FOO", "message": "BAR"}]}
 
-        self.assertRaises(AnityaPluginException, backend.parse_json, json, project)
+        with self.assertRaises(AnityaPluginException) as excinfo:
+            backend.parse_json(json, self.project)
+
+        self.assertIn('"FOO": "BAR"', str(excinfo.exception))
 
+    def test_parse_json_threshold_exceeded(self):
+        """Test behavior when rate limit threshold is exceeded."""
         # Limit reached
         json = {
             "data": {
@@ -563,21 +585,11 @@ def test_parse_json(self):
                 },
             }
         }
-        self.assertRaises(RateLimitException, backend.parse_json, json, project)
-
-        json = {
-            "data": {
-                "repository": {
-                    "refs": {"totalCount": 1, "edges": [{"node": {"name": "1.0"}}]}
-                },
-                "rateLimit": {"limit": 5000, "remaining": 5000, "resetAt": "dummy"},
-            }
-        }
-        exp = [{"cursor": None, "version": "1.0"}]
-        obs = backend.parse_json(json, project)
-        self.assertEqual(exp, obs)
+        with self.assertRaises(RateLimitException):
+            backend.parse_json(json, self.project)
+        self.assertEqual(backend.reset_time, "2008-09-03T20:56:35.450686")
 
-    def test_parse_json_threshold_reach(self):
+    def test_parse_json_threshold_reached(self):
         """
         Assert that exception is thrown when
         rate limit threshold is reached.
@@ -598,7 +610,8 @@ def test_parse_json_threshold_reach(self):
                 },
             }
         }
-        self.assertRaises(RateLimitException, backend.parse_json, json, project)
+        with self.assertRaises(RateLimitException):
+            backend.parse_json(json, project)
         self.assertEqual(backend.reset_time, "2008-09-03T20:56:35.450686")
 
 

anitya/tests/lib/test_utilities.py

@@ -314,6 +314,29 @@ def test_check_project_release_prefix_remove(self, mock_method):
         self.assertEqual(versions[0].version, "v1.0.0")
         self.assertEqual(project.latest_version, "1.0.0")
 
+    @mock.patch(
+        "anitya.lib.backends.github.GithubBackend.get_versions",
+        return_value=[
+            {"version": "1.0.0", "cursor": "Hbgf"},
+            {"version": "0.9.9", "cursor": "Hbge"},
+            {"version": "0.9.8", "cursor": "Hbgd"},
+        ],
+    )
+    def test_check_project_release_versions_with_cursor(self, mock_method):
+        """Test check_project_release() with versions with cursor."""
+        with fml_testing.mock_sends(anitya_schema.ProjectCreated):
+            project = utilities.create_project(
+                self.session,
+                name="project_name",
+                homepage="https://not-a-real-homepage.com",
+                backend="GitHub",
+                user_id="[email protected]",
+            )
+        with fml_testing.mock_sends(anitya_schema.ProjectVersionUpdated):
+            utilities.check_project_release(project, self.session)
+
+        self.assertEqual(project.latest_version_cursor, "Hbgf")
+
     @mock.patch(
         "anitya.lib.backends.npmjs.NpmjsBackend.get_versions",
         return_value=["1.0.0", "0.9.9", "0.9.8"],