From 2665e778d9a193b8714ac4bed4dc0161f7dac808 Mon Sep 17 00:00:00 2001 From: David Robertson Date: Wed, 6 Oct 2021 19:16:01 +0100 Subject: [PATCH 01/15] Alphabetise mypy `ignore-missing-imports` section --- mypy.ini | 61 ++++++++++++++++++++++++++++---------------------------- 1 file changed, 31 insertions(+), 30 deletions(-) diff --git a/mypy.ini b/mypy.ini index 86459bdcb62d..e669b0099590 100644 --- a/mypy.ini +++ b/mypy.ini @@ -192,98 +192,99 @@ disallow_untyped_defs = True [mypy-tests.storage.test_user_directory] disallow_untyped_defs = True -[mypy-pymacaroons.*] -ignore_missing_imports = True -[mypy-zope] +[mypy-authlib.*] ignore_missing_imports = True [mypy-bcrypt] ignore_missing_imports = True -[mypy-constantly] +[mypy-canonicaljson] ignore_missing_imports = True -[mypy-twisted.*] +[mypy-constantly] ignore_missing_imports = True -[mypy-treq.*] +[mypy-daemonize] ignore_missing_imports = True -[mypy-hyperlink] +[mypy-h11] ignore_missing_imports = True -[mypy-h11] +[mypy-hiredis] ignore_missing_imports = True -[mypy-msgpack] +[mypy-hyperlink] ignore_missing_imports = True -[mypy-opentracing] +[mypy-ijson.*] ignore_missing_imports = True -[mypy-OpenSSL.*] +[mypy-jaeger_client.*] ignore_missing_imports = True -[mypy-netaddr] +[mypy-josepy.*] ignore_missing_imports = True -[mypy-saml2.*] +[mypy-jsonschema] ignore_missing_imports = True -[mypy-canonicaljson] +[mypy-jwt.*] ignore_missing_imports = True -[mypy-jaeger_client.*] +[mypy-lxml] ignore_missing_imports = True -[mypy-jsonschema] +[mypy-msgpack] ignore_missing_imports = True -[mypy-signedjson.*] +[mypy-nacl.*] ignore_missing_imports = True -[mypy-prometheus_client.*] +[mypy-netaddr] ignore_missing_imports = True -[mypy-service_identity.*] +[mypy-OpenSSL.*] ignore_missing_imports = True -[mypy-daemonize] +[mypy-opentracing] ignore_missing_imports = True -[mypy-sentry_sdk] +[mypy-phonenumbers.*] ignore_missing_imports = True [mypy-PIL.*] ignore_missing_imports = True -[mypy-lxml] +[mypy-prometheus_client.*] ignore_missing_imports = True -[mypy-jwt.*] +[mypy-pymacaroons.*] ignore_missing_imports = True -[mypy-authlib.*] +[mypy-pympler.*] ignore_missing_imports = True [mypy-rust_python_jaeger_reporter.*] ignore_missing_imports = True -[mypy-nacl.*] +[mypy-saml2.*] ignore_missing_imports = True -[mypy-hiredis] +[mypy-sentry_sdk] ignore_missing_imports = True -[mypy-josepy.*] +[mypy-service_identity.*] ignore_missing_imports = True -[mypy-pympler.*] +[mypy-signedjson.*] ignore_missing_imports = True -[mypy-phonenumbers.*] +[mypy-treq.*] ignore_missing_imports = True -[mypy-ijson.*] +[mypy-twisted.*] +ignore_missing_imports = True + +[mypy-zope] ignore_missing_imports = True From 05bd37099c8cd5ebdae19c1a2e6b937908f33e85 Mon Sep 17 00:00:00 2001 From: David Robertson Date: Wed, 6 Oct 2021 19:19:24 +0100 Subject: [PATCH 02/15] typeshed has jsonschema stubs since https://github.com/python/typeshed/pull/5784 --- mypy.ini | 3 --- setup.py | 7 ++++++- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/mypy.ini b/mypy.ini index e669b0099590..b379ee0e6dcd 100644 --- a/mypy.ini +++ b/mypy.ini @@ -226,9 +226,6 @@ ignore_missing_imports = True [mypy-josepy.*] ignore_missing_imports = True -[mypy-jsonschema] -ignore_missing_imports = True - [mypy-jwt.*] ignore_missing_imports = True diff --git a/setup.py b/setup.py index c47856351081..a35a42beb16b 100755 --- a/setup.py +++ b/setup.py @@ -112,7 +112,12 @@ def exec_file(path_segments): "pygithub==1.55", ] -CONDITIONAL_REQUIREMENTS["mypy"] = ["mypy==0.812", "mypy-zope==0.2.13"] +CONDITIONAL_REQUIREMENTS["mypy"] = [ + "mypy==0.812", + "mypy-zope==0.2.13", + "types-jsonschema>=3.2.0", + +] # Dependencies which are exclusively required by unit test code. This is # NOT a list of all modules that are necessary to run the unit tests. From faa1f101d903ae721faf123976c0efa987df4b85 Mon Sep 17 00:00:00 2001 From: David Robertson Date: Wed, 6 Oct 2021 19:31:39 +0100 Subject: [PATCH 03/15] pyOpenSSL has type stubs since at least here: https://github.com/python/typeshed/pull/5649 Two fixups to keep mypy happy. The first handles that `get_notAfter` could return None. Before this would raise a generic `AttributeError`; now it raises a ValueError with specific message. The second ensures the callback to `set_verify` returns bool, not NoneType. (AFAICS this was fine because PyOpenSSL only ever used the truthiness of the callback's return value.) --- mypy.ini | 3 --- setup.py | 2 +- synapse/config/tls.py | 9 ++++++--- synapse/http/client.py | 2 +- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/mypy.ini b/mypy.ini index b379ee0e6dcd..3adae6b1c186 100644 --- a/mypy.ini +++ b/mypy.ini @@ -241,9 +241,6 @@ ignore_missing_imports = True [mypy-netaddr] ignore_missing_imports = True -[mypy-OpenSSL.*] -ignore_missing_imports = True - [mypy-opentracing] ignore_missing_imports = True diff --git a/setup.py b/setup.py index a35a42beb16b..a45aeda41604 100755 --- a/setup.py +++ b/setup.py @@ -116,7 +116,7 @@ def exec_file(path_segments): "mypy==0.812", "mypy-zope==0.2.13", "types-jsonschema>=3.2.0", - + "types-pyOpenSSL>=20.0.7", ] # Dependencies which are exclusively required by unit test code. This is diff --git a/synapse/config/tls.py b/synapse/config/tls.py index 5679f05e4270..6227434bac68 100644 --- a/synapse/config/tls.py +++ b/synapse/config/tls.py @@ -172,9 +172,12 @@ def is_disk_cert_valid(self, allow_self_signed=True): ) # YYYYMMDDhhmmssZ -- in UTC - expires_on = datetime.strptime( - tls_certificate.get_notAfter().decode("ascii"), "%Y%m%d%H%M%SZ" - ) + expiry_data = tls_certificate.get_notAfter() + if expiry_data is None: + raise ValueError( + "TLS Certificate has no expiry date, and this is not permitted" + ) + expires_on = datetime.strptime(expiry_data.decode("ascii"), "%Y%m%d%H%M%SZ") now = datetime.utcnow() days_remaining = (expires_on - now).days return days_remaining diff --git a/synapse/http/client.py b/synapse/http/client.py index 5204c3d08ccf..b5a2d333a6ce 100644 --- a/synapse/http/client.py +++ b/synapse/http/client.py @@ -912,7 +912,7 @@ class InsecureInterceptableContextFactory(ssl.ContextFactory): def __init__(self): self._context = SSL.Context(SSL.SSLv23_METHOD) - self._context.set_verify(VERIFY_NONE, lambda *_: None) + self._context.set_verify(VERIFY_NONE, lambda *_: False) def getContext(self, hostname=None, port=None): return self._context From db6001f922c64be00277932a4c86d037920b35ad Mon Sep 17 00:00:00 2001 From: David Robertson Date: Wed, 6 Oct 2021 20:12:36 +0100 Subject: [PATCH 04/15] Pillow has type stubs Had to make some more invasive changes to our media code. I replaced our naughty calls to `_getdecoder` with calls to [check_codec], introduced about seven years ago. [check_codec]: https://github.com/python-pillow/Pillow/commit/799e8312cb6ede52d83d96f9ae414fb8ba457154 mypy didn't like our use of `_getexif`. The [Pillow 3.1.0 release notes][pillow-3.1] describe this as > private, experimental, but generally widely used There's a public `getexif()` without an underscore, but that's only available since [Pillow 6.0.0][pillow-6] [pillow-3.1]: https://pillow.readthedocs.io/en/stable/releasenotes/3.1.0.html#jpegimageplugin-getexif [pillow-6]: https://pillow.readthedocs.io/en/stable/releasenotes/6.0.0.html#added-exif-class --- mypy.ini | 3 --- setup.py | 1 + synapse/rest/media/v1/__init__.py | 38 ++++++++++------------------ synapse/rest/media/v1/thumbnailer.py | 16 +++++++++--- 4 files changed, 26 insertions(+), 32 deletions(-) diff --git a/mypy.ini b/mypy.ini index 3adae6b1c186..0ae2b894a9c6 100644 --- a/mypy.ini +++ b/mypy.ini @@ -247,9 +247,6 @@ ignore_missing_imports = True [mypy-phonenumbers.*] ignore_missing_imports = True -[mypy-PIL.*] -ignore_missing_imports = True - [mypy-prometheus_client.*] ignore_missing_imports = True diff --git a/setup.py b/setup.py index a45aeda41604..e0ba21a68eec 100755 --- a/setup.py +++ b/setup.py @@ -117,6 +117,7 @@ def exec_file(path_segments): "mypy-zope==0.2.13", "types-jsonschema>=3.2.0", "types-pyOpenSSL>=20.0.7", + "types-Pillow>=8.3.4", ] # Dependencies which are exclusively required by unit test code. This is diff --git a/synapse/rest/media/v1/__init__.py b/synapse/rest/media/v1/__init__.py index 3dd16d4bb542..d5b74cddf125 100644 --- a/synapse/rest/media/v1/__init__.py +++ b/synapse/rest/media/v1/__init__.py @@ -12,33 +12,21 @@ # See the License for the specific language governing permissions and # limitations under the License. -import PIL.Image +from PIL.features import check_codec # check for JPEG support. -try: - PIL.Image._getdecoder("rgb", "jpeg", None) -except OSError as e: - if str(e).startswith("decoder jpeg not available"): - raise Exception( - "FATAL: jpeg codec not supported. Install pillow correctly! " - " 'sudo apt-get install libjpeg-dev' then 'pip uninstall pillow &&" - " pip install pillow --user'" - ) -except Exception: - # any other exception is fine - pass +if not check_codec("jpg"): + raise Exception( + "FATAL: jpeg codec not supported. Install pillow correctly! " + " 'sudo apt-get install libjpeg-dev' then 'pip uninstall pillow &&" + " pip install pillow --user'" + ) # check for PNG support. -try: - PIL.Image._getdecoder("rgb", "zip", None) -except OSError as e: - if str(e).startswith("decoder zip not available"): - raise Exception( - "FATAL: zip codec not supported. Install pillow correctly! " - " 'sudo apt-get install libjpeg-dev' then 'pip uninstall pillow &&" - " pip install pillow --user'" - ) -except Exception: - # any other exception is fine - pass +if not check_codec("zlib"): + raise Exception( + "FATAL: zip codec not supported. Install pillow correctly! " + " 'sudo apt-get install libjpeg-dev' then 'pip uninstall pillow &&" + " pip install pillow --user'" + ) diff --git a/synapse/rest/media/v1/thumbnailer.py b/synapse/rest/media/v1/thumbnailer.py index df54a4064990..0955e089701e 100644 --- a/synapse/rest/media/v1/thumbnailer.py +++ b/synapse/rest/media/v1/thumbnailer.py @@ -61,9 +61,14 @@ def __init__(self, input_path: str): self.transpose_method = None try: # We don't use ImageOps.exif_transpose since it crashes with big EXIF - image_exif = self.image._getexif() + # Safety: Pillow seems to acknowledge that this method is + # "private, experimental, but generally widely used". Pillow 6 + # includes a public getexif() method (no underscore) that we might + # consider using? + image_exif = self.image._getexif() # type: ignore if image_exif is not None: image_orientation = image_exif.get(EXIF_ORIENTATION_TAG) + assert isinstance(image_orientation, int) self.transpose_method = EXIF_TRANSPOSE_MAPPINGS.get(image_orientation) except Exception as e: # A lot of parsing errors can happen when parsing EXIF @@ -76,7 +81,10 @@ def transpose(self) -> Tuple[int, int]: A tuple containing the new image size in pixels as (width, height). """ if self.transpose_method is not None: - self.image = self.image.transpose(self.transpose_method) + # Safety: `transpose` takes an int rather than e.g. an IntEnum. + # self.transpose_method is set above to be a value in + # EXIF_TRANSPOSE_MAPPINGS, and that only contains correct values. + self.image = self.image.transpose(self.transpose_method) # type: ignore[arg-type] self.width, self.height = self.image.size self.transpose_method = None # We don't need EXIF any more @@ -101,7 +109,7 @@ def aspect(self, max_width: int, max_height: int) -> Tuple[int, int]: else: return (max_height * self.width) // self.height, max_height - def _resize(self, width: int, height: int) -> Image: + def _resize(self, width: int, height: int) -> Image.Image: # 1-bit or 8-bit color palette images need converting to RGB # otherwise they will be scaled using nearest neighbour which # looks awful. @@ -151,7 +159,7 @@ def crop(self, width: int, height: int, output_type: str) -> BytesIO: cropped = scaled_image.crop((crop_left, 0, crop_right, height)) return self._encode_image(cropped, output_type) - def _encode_image(self, output_image: Image, output_type: str) -> BytesIO: + def _encode_image(self, output_image: Image.Image, output_type: str) -> BytesIO: output_bytes_io = BytesIO() fmt = self.FORMATS[output_type] if fmt == "JPEG": From e4a27c1524542ae75b0697dc5815c987e7de3a47 Mon Sep 17 00:00:00 2001 From: David Robertson Date: Wed, 6 Oct 2021 20:26:34 +0100 Subject: [PATCH 05/15] Comment about the possibility of stubs existing --- mypy.ini | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/mypy.ini b/mypy.ini index 0ae2b894a9c6..76b29205f9d7 100644 --- a/mypy.ini +++ b/mypy.ini @@ -192,6 +192,13 @@ disallow_untyped_defs = True [mypy-tests.storage.test_user_directory] disallow_untyped_defs = True +;; Dependencies without annotations +;; Before ignoring a module, check to see if type stubs are available. +;; The `typeshed` project maintains stubs here: +;; https://github.com/python/typeshed/tree/master/stubs +;; and for each pacakge `foo` there's a corresponding `types-foo` package on PyPI, +;; which we can pull in as a dev dependency by adding to `setup.py`'s +;; `CONDITIONAL_REQUIREMENTS["mypy"]` list. [mypy-authlib.*] ignore_missing_imports = True From eec2cb083e6dc74c2de5e9b12c0af0fcd1110782 Mon Sep 17 00:00:00 2001 From: David Robertson Date: Wed, 6 Oct 2021 20:41:16 +0100 Subject: [PATCH 06/15] Changelog --- changelog.d/11006.misc | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/11006.misc diff --git a/changelog.d/11006.misc b/changelog.d/11006.misc new file mode 100644 index 000000000000..959285ef8b38 --- /dev/null +++ b/changelog.d/11006.misc @@ -0,0 +1 @@ +Use type stubs for jsonschema, pyOpenSSL and Pillow when running mypy in CI. \ No newline at end of file From df7ffe539be58ffe6a7c86c2c246191cd0430393 Mon Sep 17 00:00:00 2001 From: David Robertson Date: Wed, 6 Oct 2021 20:41:48 +0100 Subject: [PATCH 07/15] pacakge->package, thanks Patrick! Co-authored-by: Patrick Cloke --- mypy.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypy.ini b/mypy.ini index 76b29205f9d7..857327cd856b 100644 --- a/mypy.ini +++ b/mypy.ini @@ -196,7 +196,7 @@ disallow_untyped_defs = True ;; Before ignoring a module, check to see if type stubs are available. ;; The `typeshed` project maintains stubs here: ;; https://github.com/python/typeshed/tree/master/stubs -;; and for each pacakge `foo` there's a corresponding `types-foo` package on PyPI, +;; and for each package `foo` there's a corresponding `types-foo` package on PyPI, ;; which we can pull in as a dev dependency by adding to `setup.py`'s ;; `CONDITIONAL_REQUIREMENTS["mypy"]` list. From 3844414952bc6963028594c05a5726bc61ffd0c0 Mon Sep 17 00:00:00 2001 From: David Robertson Date: Thu, 7 Oct 2021 11:43:22 +0100 Subject: [PATCH 08/15] Bump mypy version and pull in additional stubs I'm not sure why newer mypy requires these---perhaps it's a consequence of pulling in a newer mypy-zope? --- setup.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index e0ba21a68eec..cea2e5c40693 100755 --- a/setup.py +++ b/setup.py @@ -113,11 +113,14 @@ def exec_file(path_segments): ] CONDITIONAL_REQUIREMENTS["mypy"] = [ - "mypy==0.812", - "mypy-zope==0.2.13", + "mypy>=0.812", + "mypy-zope>=0.2.13", + "types-bleach>=4.1.0", "types-jsonschema>=3.2.0", - "types-pyOpenSSL>=20.0.7", "types-Pillow>=8.3.4", + "types-pyOpenSSL>=20.0.7", + "types-PyYAML>=5.4.10", + "types-setuptools>=57.4.0", ] # Dependencies which are exclusively required by unit test code. This is From ec73d780a6ea97decf17654549f40445f97eac00 Mon Sep 17 00:00:00 2001 From: David Robertson Date: Thu, 7 Oct 2021 11:48:08 +0100 Subject: [PATCH 09/15] Make mypy happy by referring to struct_rusage I think struct_rusage has existed for a couple of decades now, see https://github.com/python/cpython/blame/bb3e0c240bc60fe08d332ff5955d54197f79751c/Modules/resource.c#L32 --- synapse/logging/context.py | 16 ++++++++-------- synapse/metrics/background_process_metrics.py | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/synapse/logging/context.py b/synapse/logging/context.py index 02e5ddd2ef2a..bdc018774381 100644 --- a/synapse/logging/context.py +++ b/synapse/logging/context.py @@ -52,7 +52,7 @@ is_thread_resource_usage_supported = True - def get_thread_resource_usage() -> "Optional[resource._RUsage]": + def get_thread_resource_usage() -> "Optional[resource.struct_rusage]": return resource.getrusage(RUSAGE_THREAD) @@ -61,7 +61,7 @@ def get_thread_resource_usage() -> "Optional[resource._RUsage]": # won't track resource usage. is_thread_resource_usage_supported = False - def get_thread_resource_usage() -> "Optional[resource._RUsage]": + def get_thread_resource_usage() -> "Optional[resource.struct_rusage]": return None @@ -226,10 +226,10 @@ def __str__(self): def copy_to(self, record): pass - def start(self, rusage: "Optional[resource._RUsage]"): + def start(self, rusage: "Optional[resource.struct_rusage]"): pass - def stop(self, rusage: "Optional[resource._RUsage]"): + def stop(self, rusage: "Optional[resource.struct_rusage]"): pass def add_database_transaction(self, duration_sec): @@ -289,7 +289,7 @@ def __init__( # The thread resource usage when the logcontext became active. None # if the context is not currently active. - self.usage_start: Optional[resource._RUsage] = None + self.usage_start: Optional[resource.struct_rusage] = None self.main_thread = get_thread_id() self.request = None @@ -410,7 +410,7 @@ def copy_to(self, record) -> None: # we also track the current scope: record.scope = self.scope - def start(self, rusage: "Optional[resource._RUsage]") -> None: + def start(self, rusage: "Optional[resource.struct_rusage]") -> None: """ Record that this logcontext is currently running. @@ -435,7 +435,7 @@ def start(self, rusage: "Optional[resource._RUsage]") -> None: else: self.usage_start = rusage - def stop(self, rusage: "Optional[resource._RUsage]") -> None: + def stop(self, rusage: "Optional[resource.struct_rusage]") -> None: """ Record that this logcontext is no longer running. @@ -490,7 +490,7 @@ def get_resource_usage(self) -> ContextResourceUsage: return res - def _get_cputime(self, current: "resource._RUsage") -> Tuple[float, float]: + def _get_cputime(self, current: "resource.struct_rusage") -> Tuple[float, float]: """Get the cpu usage time between start() and the given rusage Args: diff --git a/synapse/metrics/background_process_metrics.py b/synapse/metrics/background_process_metrics.py index 3a14260752ed..2ab599a33479 100644 --- a/synapse/metrics/background_process_metrics.py +++ b/synapse/metrics/background_process_metrics.py @@ -265,7 +265,7 @@ def __init__(self, name: str, instance_id: Optional[Union[int, str]] = None): super().__init__("%s-%s" % (name, instance_id)) self._proc = _BackgroundProcess(name, self) - def start(self, rusage: "Optional[resource._RUsage]"): + def start(self, rusage: "Optional[resource.struct_rusage]"): """Log context has started running (again).""" super().start(rusage) From e3e87568ca777206e0bc961b51a8d4606fb0ad7c Mon Sep 17 00:00:00 2001 From: David Robertson Date: Thu, 7 Oct 2021 11:57:17 +0100 Subject: [PATCH 10/15] Workaround https://github.com/python/mypy/issues/6463 --- synapse/push/mailer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/push/mailer.py b/synapse/push/mailer.py index e38e3c5d44e6..ce299ba3da16 100644 --- a/synapse/push/mailer.py +++ b/synapse/push/mailer.py @@ -892,7 +892,7 @@ def safe_text(raw_text: str) -> jinja2.Markup: A Markup object ready to safely use in a Jinja template. """ return jinja2.Markup( - bleach.linkify(bleach.clean(raw_text, tags=[], attributes={}, strip=False)) + bleach.linkify(bleach.clean(raw_text, tags=[], attributes=[], strip=False)) ) From 4d0bc243ed0f3f3bf7f973f05a6826f56bcc8284 Mon Sep 17 00:00:00 2001 From: David Robertson Date: Thu, 7 Oct 2021 12:05:04 +0100 Subject: [PATCH 11/15] Other mypy fixes --- synapse/storage/prepare_database.py | 1 + synapse/util/__init__.py | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/synapse/storage/prepare_database.py b/synapse/storage/prepare_database.py index a63eaddfdc1c..5963e985ab4a 100644 --- a/synapse/storage/prepare_database.py +++ b/synapse/storage/prepare_database.py @@ -487,6 +487,7 @@ def _upgrade_existing_database( spec = importlib.util.spec_from_file_location( module_name, absolute_path ) + assert spec is not None module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) # type: ignore diff --git a/synapse/util/__init__.py b/synapse/util/__init__.py index 64daff59df0d..abf53d149dba 100644 --- a/synapse/util/__init__.py +++ b/synapse/util/__init__.py @@ -51,7 +51,10 @@ def _handle_frozendict(obj: Any) -> Dict[Any, Any]: # fishing the protected dict out of the object is a bit nasty, # but we don't really want the overhead of copying the dict. try: - return obj._dict + # Safety: we catch the AttributeError immediately below. + # See https://github.com/matrix-org/python-canonicaljson/issues/36#issuecomment-927816293 + # for discussion on how frozendict's internals have changed over time. + return obj._dict # type: ignore[attr-defined] except AttributeError: # When the C implementation of frozendict is used, # there isn't a `_dict` attribute with a dict From 79fcc8ccf260eac206a03f277620f6d55332118d Mon Sep 17 00:00:00 2001 From: David Robertson Date: Thu, 7 Oct 2021 14:41:41 +0100 Subject: [PATCH 12/15] Pin mypy versions --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index cea2e5c40693..f8b4487bc19b 100755 --- a/setup.py +++ b/setup.py @@ -113,8 +113,8 @@ def exec_file(path_segments): ] CONDITIONAL_REQUIREMENTS["mypy"] = [ - "mypy>=0.812", - "mypy-zope>=0.2.13", + "mypy==0.910", + "mypy-zope==0.3.2", "types-bleach>=4.1.0", "types-jsonschema>=3.2.0", "types-Pillow>=8.3.4", From e2ec76b73dbd6818745c9fbd2d26e5b44b49af65 Mon Sep 17 00:00:00 2001 From: David Robertson Date: Fri, 8 Oct 2021 11:57:34 +0100 Subject: [PATCH 13/15] Update changelog --- changelog.d/11006.misc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changelog.d/11006.misc b/changelog.d/11006.misc index 959285ef8b38..7b4abae76a63 100644 --- a/changelog.d/11006.misc +++ b/changelog.d/11006.misc @@ -1 +1 @@ -Use type stubs for jsonschema, pyOpenSSL and Pillow when running mypy in CI. \ No newline at end of file +Bump mypy version for CI to 0.910, and pull in new type stubs for dependencies. \ No newline at end of file From 57ba17db36f845ad4b393fd8264e84e67cf01d41 Mon Sep 17 00:00:00 2001 From: David Robertson Date: Fri, 8 Oct 2021 12:09:12 +0100 Subject: [PATCH 14/15] Explicit error for `spec is None` --- synapse/storage/prepare_database.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/synapse/storage/prepare_database.py b/synapse/storage/prepare_database.py index 5963e985ab4a..11ca47ea2825 100644 --- a/synapse/storage/prepare_database.py +++ b/synapse/storage/prepare_database.py @@ -487,7 +487,10 @@ def _upgrade_existing_database( spec = importlib.util.spec_from_file_location( module_name, absolute_path ) - assert spec is not None + if spec is None: + raise RuntimeError( + f"Could not build a module spec for {module_name} at {absolute_path}" + ) module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) # type: ignore From 6630fa2c46cf5f521ebde54f9882b45b6c0e9cf6 Mon Sep 17 00:00:00 2001 From: David Robertson Date: Fri, 8 Oct 2021 12:42:48 +0100 Subject: [PATCH 15/15] Comment about Debian buster --- synapse/rest/media/v1/thumbnailer.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/synapse/rest/media/v1/thumbnailer.py b/synapse/rest/media/v1/thumbnailer.py index 0955e089701e..46701a8b8364 100644 --- a/synapse/rest/media/v1/thumbnailer.py +++ b/synapse/rest/media/v1/thumbnailer.py @@ -61,10 +61,15 @@ def __init__(self, input_path: str): self.transpose_method = None try: # We don't use ImageOps.exif_transpose since it crashes with big EXIF - # Safety: Pillow seems to acknowledge that this method is + # + # Ignore safety: Pillow seems to acknowledge that this method is # "private, experimental, but generally widely used". Pillow 6 # includes a public getexif() method (no underscore) that we might - # consider using? + # consider using instead when we can bump that dependency. + # + # At the time of writing, Debian buster (currently oldstable) + # provides version 5.4.1. It's expected to EOL in mid-2022, see + # https://wiki.debian.org/DebianReleases#Production_Releases image_exif = self.image._getexif() # type: ignore if image_exif is not None: image_orientation = image_exif.get(EXIF_ORIENTATION_TAG)