Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Provenance retrieval route #16778

Merged
merged 19 commits into from
Oct 3, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion requirements/main.in
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,6 @@ requests
requests-aws4auth
redis>=2.8.0,<6.0.0
rfc3986
rfc8785
sentry-sdk
setuptools
sigstore~=3.3.0
Expand Down
9 changes: 9 additions & 0 deletions tests/common/db/packaging.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
JournalEntry,
ProhibitedProjectName,
Project,
Provenance,
Release,
Role,
RoleInvitation,
Expand Down Expand Up @@ -142,6 +143,14 @@ class Meta:
)


class ProvenanceFactory(WarehouseFactory):
class Meta:
model = Provenance

file = factory.SubFactory(FileFactory)
provenance = factory.Faker("json")


class FileEventFactory(WarehouseFactory):
class Meta:
model = File.Event
Expand Down
10 changes: 10 additions & 0 deletions tests/conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,8 @@
from warehouse.accounts import services as account_services
from warehouse.accounts.interfaces import ITokenService, IUserService
from warehouse.admin.flags import AdminFlag, AdminFlagValue
from warehouse.attestations import services as attestations_services
from warehouse.attestations.interfaces import IIntegrityService
from warehouse.email import services as email_services
from warehouse.email.interfaces import IEmailSender
from warehouse.helpdesk import services as helpdesk_services
Expand Down Expand Up @@ -174,6 +176,7 @@ def pyramid_services(
project_service,
github_oidc_service,
activestate_oidc_service,
integrity_service,
macaroon_service,
helpdesk_service,
):
Expand All @@ -195,6 +198,7 @@ def pyramid_services(
services.register_service(
activestate_oidc_service, IOIDCPublisherService, None, name="activestate"
)
services.register_service(integrity_service, IIntegrityService, None)
services.register_service(macaroon_service, IMacaroonService, None, name="")
services.register_service(helpdesk_service, IHelpDeskService, None)

Expand Down Expand Up @@ -326,6 +330,7 @@ def get_app_config(database, nondefaults=None):
"docs.backend": "warehouse.packaging.services.LocalDocsStorage",
"sponsorlogos.backend": "warehouse.admin.services.LocalSponsorLogoStorage",
"billing.backend": "warehouse.subscriptions.services.MockStripeBillingService",
"integrity.backend": "warehouse.attestations.services.NullIntegrityService",
"billing.api_base": "http://stripe:12111",
"billing.api_version": "2020-08-27",
"mail.backend": "warehouse.email.services.SMTPEmailSender",
Expand Down Expand Up @@ -557,6 +562,11 @@ def dummy_attestation():
)


@pytest.fixture
def integrity_service(db_session):
return attestations_services.NullIntegrityService(db_session)


@pytest.fixture
def macaroon_service(db_session):
return macaroon_services.DatabaseMacaroonService(db_session)
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
{
"version": 1,
"verification_material": {
"certificate": "MIIC6zCCAnGgAwIBAgIUFgmhIYx8gvBGePCTacG/4kbBdRwwCgYIKoZIzj0EAwMwNzEVMBMGA1UEChMMc2lnc3RvcmUuZGV2MR4wHAYDVQQDExVzaWdzdG9yZS1pbnRlcm1lZGlhdGUwHhcNMjQwODI5MTcwOTM5WhcNMjQwODI5MTcxOTM5WjAAMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEtGrMPml4OtsRJ3Z6qRahs0kHCZxP4n9fvrJE957WVxgAGg4k6a1PbRJY9nT9wKpRrZmKV++AgA9ndhdruXXaAKOCAZAwggGMMA4GA1UdDwEB/wQEAwIHgDATBgNVHSUEDDAKBggrBgEFBQcDAzAdBgNVHQ4EFgQUosNvhYEuTPfgyU/dZfu93lFGRNswHwYDVR0jBBgwFoAU39Ppz1YkEZb5qNjpKFWixi4YZD8wQAYDVR0RAQH/BDYwNIEyOTE5NDM2MTU4MjM2LWNvbXB1dGVAZGV2ZWxvcGVyLmdzZXJ2aWNlYWNjb3VudC5jb20wKQYKKwYBBAGDvzABAQQbaHR0cHM6Ly9hY2NvdW50cy5nb29nbGUuY29tMCsGCisGAQQBg78wAQgEHQwbaHR0cHM6Ly9hY2NvdW50cy5nb29nbGUuY29tMIGKBgorBgEEAdZ5AgQCBHwEegB4AHYA3T0wasbHETJjGR4cmWc3AqJKXrjePK3/h4pygC8p7o4AAAGRnx0/aQAABAMARzBFAiBogvcKHIIR9FcX1vQgDhGtAl0XQoMRiEB3OdUWO94P1gIhANdJlyISdtvVrHes25dWKTLepy+IzQmzfQU/S7cxWHmOMAoGCCqGSM49BAMDA2gAMGUCMGe2xTiuenbjdt1d2e4IaCiwRh2G4KAtyujRESSSUbpuGme/o9ouiApeONBv2CvvGAIxAOEkAGFO3aALE3IPNosxqaz9MbqJOdmYhB1Cz1D7xbFc/m243VxJWxaC/uOFEpyiYQ==",
"transparency_entries": [
{
"logIndex": "125970014",
"logId": {
"keyId": "wNI9atQGlz+VWfO6LRygH4QUfY/8W4RFwiT5i5WRgB0="
},
"kindVersion": {
"kind": "dsse",
"version": "0.0.1"
},
"integratedTime": "1724951379",
"inclusionPromise": {
"signedEntryTimestamp": "MEUCIQCHrKFTeXNY432S0bUSBS69S8d5JnNcDXa41q6OEvxEwgIgaZstc5Jpm0IgwFC7RDTXYEAKk+3aG/MkRkaPdJdyn8U="
},
"inclusionProof": {
"logIndex": "4065752",
"rootHash": "7jVDF3UNUZVEU85ffETQ3WKfXhOoMi4cgytJM250HTk=",
"treeSize": "4065754",
"hashes": [
"NwJgWJoxjearbnEIT9bnWXpzo0LGNrR1cpWId0g66rE=",
"kLjpW3Eh7pQJNOvyntghzF57tcfqk2IzX7cqiBDgGf8=",
"FW8y9LQ1i3q+MnbeGJipKGl4VfX1zRBOD7TmhbEw7uI=",
"mKcbGJDJ/+buNbXy9Eyv94nVoAyUauuIlN3cJg3qSBY=",
"5VytqqAHhfRkRWMrY43UXWCnRBb7JwElMlKpY5JueBc=",
"mZJnD39LTKdis2wUTz1OOMx3r7HwgJh9rnb2VwiPzts=",
"MXZOQFJFiOjREF0xwMOCXu29HwTchjTtl/BeFoI51wY=",
"g8zCkHnLwO3LojK7g5AnqE8ezSNRnCSz9nCL5GD3a8A=",
"RrZsD/RSxNoujlvq/MsCEvLSkKZfv0jmQM9Kp7qbJec=",
"QxmVWsbTp4cClxuAkuT51UH2EY7peHMVGKq7+b+cGwQ=",
"Q2LAtNzOUh+3PfwfMyNxYb06fTQmF3VeTT6Fr6Upvfc=",
"ftwAu6v62WFDoDmcZ1JKfrRPrvuiIw5v3BvRsgQj7N8="
],
"checkpoint": {
"envelope": "rekor.sigstore.dev - 1193050959916656506\n4065754\n7jVDF3UNUZVEU85ffETQ3WKfXhOoMi4cgytJM250HTk=\n\n— rekor.sigstore.dev wNI9ajBGAiEAhMomhZHOTNB5CVPO98CMXCv01ZlIF+C+CgzraAB01r8CIQCEuXbv6aqguUpB/ig5eXRIbarvxLXkg3nX48DzambktQ==\n"
}
},
"canonicalizedBody": "eyJhcGlWZXJzaW9uIjoiMC4wLjEiLCJraW5kIjoiZHNzZSIsInNwZWMiOnsiZW52ZWxvcGVIYXNoIjp7ImFsZ29yaXRobSI6InNoYTI1NiIsInZhbHVlIjoiOWRiNGJjMzE3MTgyZWI3NzljNDIyY2Q0NGI2ZDdlYTk5ZWM1M2Q3M2JiY2ZjZWVmZTIyNWVlYjQ3NTQyMjc4OCJ9LCJwYXlsb2FkSGFzaCI6eyJhbGdvcml0aG0iOiJzaGEyNTYiLCJ2YWx1ZSI6IjlkYjY0MjlhOTkzZGFiYTI4NzAwODk2ZTY2MzNjNzkxYWE0MDM3ODQ4NjJiYzY2MDBkM2E4NjYwMGQzYjA1NjMifSwic2lnbmF0dXJlcyI6W3sic2lnbmF0dXJlIjoiTUVVQ0lCaGlOL25NR0w3aHpZQk9QQjlUTGtuaEdTZEtuQ0Q0ekI3TDV5ZXc0QmJ3QWlFQXJzOHl6MCtCT2NnSEtzS0JzTXVOeVlhREdaRTBVV0JuMEdwNVpGMzUvU2M9IiwidmVyaWZpZXIiOiJMUzB0TFMxQ1JVZEpUaUJEUlZKVVNVWkpRMEZVUlMwdExTMHRDazFKU1VNMmVrTkRRVzVIWjBGM1NVSkJaMGxWUm1kdGFFbFplRGhuZGtKSFpWQkRWR0ZqUnk4MGEySkNaRkozZDBObldVbExiMXBKZW1vd1JVRjNUWGNLVG5wRlZrMUNUVWRCTVZWRlEyaE5UV015Ykc1ak0xSjJZMjFWZFZwSFZqSk5ValIzU0VGWlJGWlJVVVJGZUZaNllWZGtlbVJIT1hsYVV6RndZbTVTYkFwamJURnNXa2RzYUdSSFZYZElhR05PVFdwUmQwOUVTVFZOVkdOM1QxUk5OVmRvWTA1TmFsRjNUMFJKTlUxVVkzaFBWRTAxVjJwQlFVMUdhM2RGZDFsSUNrdHZXa2w2YWpCRFFWRlpTVXR2V2tsNmFqQkVRVkZqUkZGblFVVjBSM0pOVUcxc05FOTBjMUpLTTFvMmNWSmhhSE13YTBoRFduaFFORzQ1Wm5aeVNrVUtPVFUzVjFaNFowRkhaelJyTm1FeFVHSlNTbGs1YmxRNWQwdHdVbkphYlV0V0t5dEJaMEU1Ym1Sb1pISjFXRmhoUVV0UFEwRmFRWGRuWjBkTlRVRTBSd3BCTVZWa1JIZEZRaTkzVVVWQmQwbElaMFJCVkVKblRsWklVMVZGUkVSQlMwSm5aM0pDWjBWR1FsRmpSRUY2UVdSQ1owNVdTRkUwUlVablVWVnZjMDUyQ21oWlJYVlVVR1puZVZVdlpGcG1kVGt6YkVaSFVrNXpkMGgzV1VSV1VqQnFRa0puZDBadlFWVXpPVkJ3ZWpGWmEwVmFZalZ4VG1wd1MwWlhhWGhwTkZrS1drUTRkMUZCV1VSV1VqQlNRVkZJTDBKRVdYZE9TVVY1VDFSRk5VNUVUVEpOVkZVMFRXcE5Na3hYVG5aaVdFSXhaRWRXUVZwSFZqSmFWM2gyWTBkV2VRcE1iV1I2V2xoS01tRlhUbXhaVjA1cVlqTldkV1JETldwaU1qQjNTMUZaUzB0M1dVSkNRVWRFZG5wQlFrRlJVV0poU0ZJd1kwaE5Oa3g1T1doWk1rNTJDbVJYTlRCamVUVnVZakk1Ym1KSFZYVlpNamwwVFVOelIwTnBjMGRCVVZGQ1p6YzRkMEZSWjBWSVVYZGlZVWhTTUdOSVRUWk1lVGxvV1RKT2RtUlhOVEFLWTNrMWJtSXlPVzVpUjFWMVdUSTVkRTFKUjB0Q1oyOXlRbWRGUlVGa1dqVkJaMUZEUWtoM1JXVm5RalJCU0ZsQk0xUXdkMkZ6WWtoRlZFcHFSMUkwWXdwdFYyTXpRWEZLUzFoeWFtVlFTek12YURSd2VXZERPSEEzYnpSQlFVRkhVbTU0TUM5aFVVRkJRa0ZOUVZKNlFrWkJhVUp2WjNaalMwaEpTVkk1Um1OWUNqRjJVV2RFYUVkMFFXd3dXRkZ2VFZKcFJVSXpUMlJWVjA4NU5GQXhaMGxvUVU1a1NteDVTVk5rZEhaV2NraGxjekkxWkZkTFZFeGxjSGtyU1hwUmJYb0tabEZWTDFNM1kzaFhTRzFQVFVGdlIwTkRjVWRUVFRRNVFrRk5SRUV5WjBGTlIxVkRUVWRsTW5oVWFYVmxibUpxWkhReFpESmxORWxoUTJsM1VtZ3lSd28wUzBGMGVYVnFVa1ZUVTFOVlluQjFSMjFsTDI4NWIzVnBRWEJsVDA1Q2RqSkRkblpIUVVsNFFVOUZhMEZIUms4ellVRk1SVE5KVUU1dmMzaHhZWG81Q2sxaWNVcFBaRzFaYUVJeFEzb3hSRGQ0WWtaakwyMHlORE5XZUVwWGVHRkRMM1ZQUmtWd2VXbFpVVDA5Q2kwdExTMHRSVTVFSUVORlVsUkpSa2xEUVZSRkxTMHRMUzBLIn1dfX0="
}
]
},
"envelope": {
"statement": "eyJfdHlwZSI6Imh0dHBzOi8vaW4tdG90by5pby9TdGF0ZW1lbnQvdjEiLCJzdWJqZWN0IjpbeyJuYW1lIjoic2FtcGxlcHJvamVjdC0zLjAuMC50YXIuZ3oiLCJkaWdlc3QiOnsic2hhMjU2IjoiMTE3ZWQ4OGU1ZGIwNzNiYjkyOTY5YTc1NDU3NDVmZDk3N2VlODViNzAxOTcwNmRkMjU2YTY0MDU4ZjcwOTYzZCJ9fV0sInByZWRpY2F0ZVR5cGUiOiJodHRwczovL2RvY3MucHlwaS5vcmcvYXR0ZXN0YXRpb25zL3B1Ymxpc2gvdjEiLCJwcmVkaWNhdGUiOm51bGx9",
"signature": "MEUCIBhiN/nMGL7hzYBOPB9TLknhGSdKnCD4zB7L5yew4BbwAiEArs8yz0+BOcgHKsKBsMuNyYaDGZE0UWBn0Gp5ZF35/Sc="
}
}
58 changes: 58 additions & 0 deletions tests/functional/api/test_integrity.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import json

from http import HTTPStatus
from pathlib import Path

from ...common.db.packaging import (
FileFactory,
ProjectFactory,
ProvenanceFactory,
ReleaseFactory,
)

_HERE = Path(__file__).parent
_ASSETS = _HERE.parent / "_fixtures"
assert _ASSETS.is_dir()


def test_provenance_available(webtest):
with open(
_ASSETS / "sampleproject-3.0.0.tar.gz.publish.attestation",
) as f:
attestation_contents = f.read()
attestation_json = json.loads(attestation_contents)

project = ProjectFactory.create()
release = ReleaseFactory.create(project=project)
file_ = FileFactory.create(release=release, packagetype="sdist")
ProvenanceFactory.create(
file=file_,
provenance={"attestation_bundles": [{"attestations": [attestation_json]}]},
)

response = webtest.get(
f"/integrity/{project.name}/{release.version}/{file_.filename}/provenance",
status=HTTPStatus.OK,
)
assert response.json
assert "attestation_bundles" in response.json
attestation_bundles = response.json["attestation_bundles"]
assert len(attestation_bundles) == 1
attestation_bundle = attestation_bundles[0]
assert "attestations" in attestation_bundle
attestations = attestation_bundle["attestations"]
assert len(attestations) == 1
attestation = attestations[0]
assert attestation == attestation_json
101 changes: 80 additions & 21 deletions tests/functional/forklift/test_legacy.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,19 +11,27 @@
# limitations under the License.

import base64
import json

from http import HTTPStatus
from pathlib import Path

import pymacaroons
import pytest

from webob.multidict import MultiDict

from tests.common.db.oidc import GitHubPublisherFactory
from tests.common.db.packaging import ProjectFactory, RoleFactory
from warehouse.macaroons import caveats

from ...common.db.accounts import UserFactory
from ...common.db.macaroons import MacaroonFactory

_HERE = Path(__file__).parent
_ASSETS = _HERE.parent / "_fixtures"
assert _ASSETS.is_dir()


def test_incorrect_post_redirect(webtest):
"""
Expand Down Expand Up @@ -68,13 +76,7 @@ def test_remove_doc_upload(webtest):
],
)
def test_file_upload(webtest, upload_url, additional_data):
user = UserFactory.create(
with_verified_primary_email=True,
password=( # 'password'
"$argon2id$v=19$m=1024,t=6,p=6$EiLE2Nsbo9S6N+acs/beGw$ccyZDCZstr1/+Y/1s3BVZ"
"HOJaqfBroT0JCieHug281c"
),
)
user = UserFactory.create(with_verified_primary_email=True, clear_pwd="password")

# Construct the macaroon
dm = MacaroonFactory.create(
Expand Down Expand Up @@ -135,13 +137,7 @@ def test_file_upload(webtest, upload_url, additional_data):


def test_duplicate_file_upload_error(webtest):
user = UserFactory.create(
with_verified_primary_email=True,
password=( # 'password'
"$argon2id$v=19$m=1024,t=6,p=6$EiLE2Nsbo9S6N+acs/beGw$ccyZDCZstr1/+Y/1s3BVZ"
"HOJaqfBroT0JCieHug281c"
),
)
user = UserFactory.create(with_verified_primary_email=True, clear_pwd="password")

# Construct the macaroon
dm = MacaroonFactory.create(
Expand Down Expand Up @@ -215,13 +211,7 @@ def test_duplicate_file_upload_error(webtest):


def test_invalid_classifier_upload_error(webtest):
user = UserFactory.create(
with_verified_primary_email=True,
password=( # 'password'
"$argon2id$v=19$m=1024,t=6,p=6$EiLE2Nsbo9S6N+acs/beGw$ccyZDCZstr1/+Y/1s3BVZ"
"HOJaqfBroT0JCieHug281c"
),
)
user = UserFactory.create(with_verified_primary_email=True, clear_pwd="password")

# Construct the macaroon
dm = MacaroonFactory.create(
Expand Down Expand Up @@ -270,3 +260,72 @@ def test_invalid_classifier_upload_error(webtest):
status=HTTPStatus.BAD_REQUEST,
)
assert "'This :: Is :: Invalid' is not a valid classifier" in resp.body.decode()


def test_provenance_upload(webtest):
user = UserFactory.create(with_verified_primary_email=True, clear_pwd="password")
project = ProjectFactory.create(name="sampleproject")
RoleFactory.create(user=user, project=project, role_name="Owner")
publisher = GitHubPublisherFactory.create(projects=[project])

# Construct the macaroon. This needs to be based on a Trusted Publisher, which is
# required to upload attestations
dm = MacaroonFactory.create(
oidc_publisher_id=publisher.id,
caveats=[
caveats.OIDCPublisher(oidc_publisher_id=str(publisher.id)),
caveats.ProjectID(project_ids=[str(p.id) for p in publisher.projects]),
],
additional={"oidc": {"ref": "someref", "sha": "somesha"}},
)

m = pymacaroons.Macaroon(
location="localhost",
identifier=str(dm.id),
key=dm.key,
version=pymacaroons.MACAROON_V2,
)
for caveat in dm.caveats:
m.add_first_party_caveat(caveats.serialize(caveat))
serialized_macaroon = f"pypi-{m.serialize()}"

with open(_ASSETS / "sampleproject-3.0.0.tar.gz", "rb") as f:
content = f.read()

with open(
_ASSETS / "sampleproject-3.0.0.tar.gz.publish.attestation",
) as f:
attestation_contents = f.read()

webtest.set_authorization(("Basic", ("__token__", serialized_macaroon)))
webtest.post(
"/legacy/?:action=file_upload",
params={
"name": "sampleproject",
"sha256_digest": (
"117ed88e5db073bb92969a7545745fd977ee85b7019706dd256a64058f70963d"
),
"filetype": "sdist",
"metadata_version": "2.1",
"version": "3.0.0",
"attestations": f"[{attestation_contents}]",
},
upload_files=[("content", "sampleproject-3.0.0.tar.gz", content)],
status=HTTPStatus.OK,
)

assert len(project.releases) == 1
release = project.releases[0]
assert release.files.count() == 1
file_ = project.releases[0].files[0]
assert file_.provenance is not None
provenance = file_.provenance.provenance
assert "attestation_bundles" in provenance
attestation_bundles = provenance["attestation_bundles"]
assert len(attestation_bundles) == 1
bundle = provenance["attestation_bundles"][0]
assert "attestations" in bundle
attestations = bundle["attestations"]
assert len(attestations) == 1
attestation = attestations[0]
assert attestation == json.loads(attestation_contents)
54 changes: 54 additions & 0 deletions tests/unit/api/test_integrity.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import pretend
import pytest

from warehouse.api import integrity


def test_select_content_type(db_request):
db_request.accept = "application/json"

assert (
integrity._select_content_type(db_request)
== integrity.MIME_PYPI_INTEGRITY_V1_JSON
)


# Backstop; can be removed/changed once this view supports HTML.
@pytest.mark.parametrize(
"content_type",
[integrity.MIME_TEXT_HTML, integrity.MIME_PYPI_INTEGRITY_V1_HTML],
)
def test_provenance_for_file_bad_accept(db_request, content_type):
db_request.accept = content_type
response = integrity.provenance_for_file(pretend.stub(), db_request)
assert response.status_code == 406
assert response.json == {"message": "Request not acceptable"}


def test_provenance_for_file_not_enabled(db_request, monkeypatch):
monkeypatch.setattr(db_request, "flags", pretend.stub(enabled=lambda *a: True))

response = integrity.provenance_for_file(pretend.stub(), db_request)
assert response.status_code == 403
assert response.json == {"message": "Attestations temporarily disabled"}


def test_provenance_for_file_not_present(db_request, monkeypatch):
monkeypatch.setattr(db_request, "flags", pretend.stub(enabled=lambda *a: False))
file = pretend.stub(provenance=None, filename="fake-1.2.3.tar.gz")

response = integrity.provenance_for_file(file, db_request)
assert response.status_code == 404
assert response.json == {"message": "No provenance available for fake-1.2.3.tar.gz"}
Loading