add update_checksup (#1072)

Author: 1ucian0

…kflows/weekly-badge-and-stats-update.yml .github/workflows/weekly-update.yml.github/workflows/weekly-badge-and-stats-update.yml renamed to .github/workflows/weekly-update.yml .github/workflows/weekly-badge-and-stats-update.yml renamed to .github/workflows/weekly-update.yml

@@ -1,4 +1,4 @@
-name: Weekly | Update stats and badges
+name: Weekly | Update member data
 
 on:
   workflow_dispatch:
@@ -7,6 +7,7 @@ on:
 
 jobs:
   batch_update_badges:
+    name: "Update members data"
     runs-on: ubuntu-latest
     steps:
     - name: Setup variables
@@ -24,27 +25,30 @@ jobs:
         python -m pip install --upgrade pip
         pip install -r requirements.txt
 
-    - name: Update GitHub stats
+    - name: Update GitHub data
       run: python manager.py members update_github
       env:
         GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
-    - name: Update PyPI stats
+    - name: Update PyPI data
       run: python manager.py members update_pypi
 
-    - name: Update Julia stats
+    - name: Update Julia data
       run: python manager.py members update_julia
 
+    - name: Update checkups data
+      run: python manager.py members update_checkups
+
     - name: Create PR for stats and badges update
       id: cpr
       uses: peter-evans/create-pull-request@v7
       with:
         author: GitHub <noreply@github.com>
-        commit-message: Badges and stats update for ${{ env.dateIseconds }}
-        title: Badges and stats update for ${{ env.date_bdY }}
+        commit-message: Member data update for ${{ env.dateIseconds }}
+        title: Member data update for ${{ env.date_bdY }}
         body: |
-          Badges and stats update
-          Time: ${{ env.dateIseconds }}
+          Member data update
+          when: ${{ env.dateIseconds }}
           ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
         branch: ${{ env.pr_branch_name }}
         labels: member update

.gitignore

@@ -140,6 +140,7 @@ dmypy.json
 
 # Generated files
 website/
+badges/
 docs/assets/
 all_qiskit_versions.json
 

ecosystem/check.py

@@ -1,5 +1,6 @@
 """Checks/Validations section."""
 
+import datetime
 from pathlib import Path
 
 import tomllib
@@ -22,17 +23,25 @@ def __init__(self, toml_filename: str = None, resources_dir: str = None):
             data = tomllib.load(f)
         self._data = data
 
-    def check(self, checkid):
+    def checkup(self, checkup_id):
         """Given an ID for a check, the details"""
-        return self._data[checkid]
+        return self._data[checkup_id]
+
+    def id_by_pytest_node(self, node_id):
+        """Given a PyTest node ID, find the test ID"""
+        for id_, checkup in self._data.items():
+            if not isinstance(checkup, dict):
+                continue
+            if checkup.get("checker") == node_id:
+                return id_
+        raise AttributeError(f"nodeid {node_id} not found")
 
 
 class CheckData(JsonSerializable):
     """
     The validation data related to a project
     """
 
-    dict_keys = ["id", "xfailed", "since", "details"]
     labels_toml = ChecksToml()
 
     def __init__(self, id_: str, xfailed=None, since=None, details=None, **_):
@@ -42,20 +51,43 @@ def __init__(self, id_: str, xfailed=None, since=None, details=None, **_):
         self.details = details
 
     def to_dict(self) -> dict:
-        ret = {}
-        if self.xfailed:
-            ret["xfailed"] = self.xfailed
-        if self.since:
-            ret["since"] = self.since
-        if self.details:
-            ret["details"] = self.details
-
+        ret = super().to_dict()
+        del ret["id"]
+        ret["importance"] = self.importance
         return ret
 
+    @property
+    def importance(self):
+        """get the importance of the check"""
+        if "importance" in self.labels_toml.checkup(self.id):
+            return self.labels_toml.checkup(self.id)["importance"]
+        return None
+
+    def __repr__(self):
+        return str(self.to_dict())
+
     def __getattr__(self, name):
-        return self.labels_toml.check(self.id)[name]
+        return self.labels_toml.checkup(self.id)[name]
 
-    def importance(self):
+    @classmethod
+    def from_report(cls, pytest_report):
+        """creates a CheckData instance based on a PyTest report"""
+        assertion_msg = (
+            pytest_report.longreprtext.partition("\n")[0].split(":", 1)[1].strip()
+        )
+        test_id = cls.labels_toml.id_by_pytest_node(pytest_report.nodeid)
+        if hasattr(pytest_report, "wasxfail") and pytest_report.wasxfail:
+            return CheckData(
+                test_id, details=assertion_msg, xfailed=pytest_report.wasxfail
+            )
+        since = (
+            pytest_report.previously_failed.kwargs["since"]
+            if hasattr(pytest_report, "previously_failed")
+            else datetime.datetime.today()
+        )
+        return CheckData(test_id, details=assertion_msg, since=since)
+
+    def importances(self):
         """Returns dict name->description with the possible importance values"""
         return {i["name"]: i["description"] for i in self.labels_toml["importance"]}
 

ecosystem/cli/members.py

@@ -1,5 +1,6 @@
 """CliMembers class for controlling all CLI functions."""
 
+import datetime
 import json
 import tomllib
 import os
@@ -161,6 +162,44 @@ def update_julia(self, name=None):
             project.update_julia()
             self.dao.update(project.name_id, julia=project.julia)
 
+    def update_checkups(self, name=None):
+        """
+        Updates checkups data.
+        If <name> is not given, runs on all the members.
+        Otherwise, all the members with name_id that contains <name>
+        as substring are checked.
+        """
+        for project in self.dao.get_all(name):
+            project.update_checkups()
+            if project.checks:
+                today = datetime.datetime.today()
+                for checkup_id, checkup in project.checks.items():
+                    if checkup.xfailed:
+                        self.logger.info(
+                            "☑️ %s expected to fail checkup %s: %s ",
+                            project.name,
+                            checkup_id,
+                            checkup.xfailed,
+                        )
+                        continue
+                    days_since = (today - checkup.since).days
+                    for_x_days = (
+                        f"for {days_since} days" if days_since else "since today"
+                    )
+                    self.logger.info(
+                        "❌ %s failed checkup %s (%s)",
+                        project.name,
+                        checkup_id,
+                        for_x_days,
+                    )
+            else:
+                self.logger.info(
+                    "✅ %s (%s) passed all the checkups",
+                    project.name,
+                    project.name_id,
+                )
+            self.dao.update(project.name_id, checks=project.checks)
+
     @staticmethod
     def filter_data(
         member_dict, data_map, forced_addition=False

ecosystem/member.py

@@ -4,12 +4,14 @@
 from uuid import uuid4
 import re
 
+from .error_handling import EcosystemError
 from .julia import JuliaData
 from .serializable import JsonSerializable, parse_datetime
 from .github import GitHubData
 from .pypi import PyPIData
 from .check import CheckData
 from .request import URL, request_json
+from .validation import validate_member
 
 
 class Member(JsonSerializable):  # pylint: disable=too-many-instance-attributes
@@ -168,8 +170,23 @@ def _create_qisk_dot_it_link_for_badge(self):
         response = request_json("https://api-ssl.bitly.com/v4/bitlinks", post=data)
         return response["link"]
 
+    def _qisk_dot_it_link_exists(self):
+        try:
+            request_json(
+                f"https://api-ssl.bitly.com/v4/bitlinks/qisk.it/e-{self.short_uuid}",
+                parser=lambda x: {},
+            )
+        except EcosystemError as error:
+            if "Not Found (404)" in error.message:
+                return None
+            if "Unauthorized (401)" in error.message:
+                return "UNAUTHORIZED"
+            raise error
+        return f"https://qisk.it/e-{self.short_uuid}"
+
     def update_badge(self):
         """If not there yet, creates a new Bitly link for the badge"""
+        self.badge = self._qisk_dot_it_link_exists()
         if self.badge is None:
             self.badge = self._create_qisk_dot_it_link_for_badge()
 
@@ -252,4 +269,17 @@ def from_submission(cls, submission, issue_number: str = None):
     @property
     def xfails(self):
         """list of xfails for a self member"""
-        return [check for check in self.checks if check.xfailed]
+        return [
+            check
+            for checkid, check in self.checks.items()
+            if hasattr(check, "xfailed") and check.xfailed
+        ]
+
+    def update_checkups(self):
+        """Runs validation tests and updates the check-ups sections"""
+        checkups = {}
+        report = validate_member(self, verbose_level="-q")
+        for test in report.xfailed + report.failed:
+            checkup_data = CheckData.from_report(test)
+            checkups[checkup_data.id] = checkup_data
+        self.checks = checkups

ecosystem/resources/checks.toml

@@ -26,7 +26,16 @@ importance = "STRONG-RECOMMENDATION"
 title = "URLs must exists"
 description = "None of the URLs must not return 404s or other forms of permanent errors. Rediractions are accepted."
 applies_to = "all"
-affects = ["member.website", "member.url", "member.documentation", "member.packages.*"]
+affects = ["member.website", "member.url", "member.documentation", "member.packages.*", "member.pypi.*.url"]
+category = "SUBMISSION"
+importance = "IMPORTANT"
+
+[013]
+title = "URLs use a secure schema"
+description = "All the URLs must use HTTTPS (HyperText Transfer Protocol Secure) schema."
+applies_to = "all"
+affects = ["member.website", "member.url", "member.documentation", "member.packages.*", "member.pypi.*.url"]
+checker = "test_url.py::TestURLs::test_http"
 category = "SUBMISSION"
 importance = "IMPORTANT"
 
@@ -35,6 +44,7 @@ title="Adhere to the Qiskit CoC"
 applies_to = "all"
 category = "SUBMISSION"
 description = "Project submitters should adhere to the Qiskit code of conduct. If the submitter is the project maintainer, they can enforce your their code of conduct in addition to the one coming from Qiskit."
+importance = "CRITICAL"
 
 [Q20]
 title = "Be compatible with the Qiskit SDK v2 or newer"

ecosystem/resources/members/arvaklite_7fcadf8b.toml

@@ -8,6 +8,7 @@ website = "https://arvak.io"
 group = "Transpiler plugin"
 packages = []
 uuid = "7fcadf8b-f043-4eac-a491-9783a1c00cfe"
+badge = "https://qisk.it/e-7fcadf8b"
 
 [github]
 url = "https://github.com/hiq-lab/arvak-lite"

ecosystem/resources/members/azurequant_8abf631e.toml

@@ -6,7 +6,7 @@ labels = [ "provider", "cloud service",]
 created_at = 1715084759.598566
 group = "Compute provider"
 uuid = "8abf631e-bbf0-4624-b9c7-60073981f5b0"
-badge = "https://qisk.it/qe-8abf631e"
+badge = "https://qisk.it/e-8abf631e"
 
 [github]
 url = "https://github.com/microsoft/azure-quantum-python"

ecosystem/resources/members/qiskitinsp_1513e7fe.toml

@@ -13,7 +13,7 @@ url = "https://github.com/neuralsorcerer/qiskit-inspect"
 owner = "neuralsorcerer"
 repo = "qiskit-inspect"
 stars = 1
-homepage = "http://pypi.org/project/qiskit-inspect/"
+homepage = "https://pypi.org/project/qiskit-inspect/"
 license = "Apache License 2.0"
 description = "Debugger, backend prefix tracing, and assertions for Qiskit"
 total_dependent_repositories = 0

ecosystem/validation/__init__.py

@@ -1,12 +1,10 @@
 """Validation module"""
 
 import pytest
+from ecosystem.validation.conftest import ValidationReport
 
 # pylint: disable=pointless-string-statement
 """
-BIG TODO:
-MOVE THE VALIDATIONS TO REGULAR PYTHON-BASED TESTS.
-THIS CUSTOM VALIDATION DISCOVERY LOOKS TOO MUCH TO UNITTEST DISCOVERY
 
 TODO json:
  - check no member repetition
@@ -34,59 +32,6 @@
 """
 
 
-class ValidationReport:
-    # pylint: disable=missing-function-docstring, missing-class-docstring
-
-    def __init__(self, member):
-        self._member = member
-        self.collected = 0
-        self.exitcode = 0
-        self.passed = []
-        self.failed = []
-        self.xfailed = []
-        self.skipped = []
-
-    @property
-    def xfails(self):
-        return {
-            checkdata.checker: checkdata.xfailed for checkdata in self._member.xfails
-        }
-
-    def pytest_itemcollected(self, item):
-        # pylint: disable=protected-access
-        item._nodeid = "/".join(item.nodeid.split("/")[2:])
-
-    @pytest.hookimpl(hookwrapper=True)
-    def pytest_runtest_makereport(self, item, call):  # pylint: disable=unused-argument
-
-        outcome = yield
-        report = outcome.get_result()
-        if report.when == "call":
-            if report.passed:
-                self.passed.append(report)
-            elif report.failed:
-                self.failed.append(report)
-            elif report.wasxfail:
-                self.xfailed.append(report)
-            else:
-                self.skipped.append(report)
-
-    def pytest_collection_modifyitems(self, items):
-        self.collected = len(items)
-        for item in items:
-            if item.nodeid in self.xfails:
-                item.add_marker(pytest.mark.xfail(reason=self.xfails[item.nodeid]))
-
-    def pytest_terminal_summary(
-        self, terminalreporter, exitstatus
-    ):  # pylint: disable=unused-argument
-        self.exitcode = exitstatus.value if hasattr(exitstatus, "value") else exitstatus
-
-    @pytest.fixture
-    def member(self):
-        return self._member
-
-
 def validate_member(member, verbose_level=None):
     """Runs all the validation for a member
     verbose_level: -v, -vv, -q