diff --git a/Misc/NEWS.d/next/Library/2022-11-02-05-53-25.gh-issue-83004.qc_KHr.rst b/Misc/NEWS.d/next/Library/2022-11-02-05-53-25.gh-issue-83004.qc_KHr.rst
new file mode 100644
index 00000000000000..de00063420638a
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2022-11-02-05-53-25.gh-issue-83004.qc_KHr.rst
@@ -0,0 +1 @@
+Clean up refleaks on failed module initialisation in in :mod:`_pickle`
diff --git a/Modules/_pickle.c b/Modules/_pickle.c
index ce54a2dbf3de34..956c7b6f816b9b 100644
--- a/Modules/_pickle.c
+++ b/Modules/_pickle.c
@@ -8027,16 +8027,15 @@ PyInit__pickle(void)
     if (st->UnpicklingError == NULL)
         return NULL;
 
-    Py_INCREF(st->PickleError);
-    if (PyModule_AddObject(m, "PickleError", st->PickleError) < 0)
+    if (PyModule_AddObjectRef(m, "PickleError", st->PickleError) < 0) {
         return NULL;
-    Py_INCREF(st->PicklingError);
-    if (PyModule_AddObject(m, "PicklingError", st->PicklingError) < 0)
+    }
+    if (PyModule_AddObjectRef(m, "PicklingError", st->PicklingError) < 0) {
         return NULL;
-    Py_INCREF(st->UnpicklingError);
-    if (PyModule_AddObject(m, "UnpicklingError", st->UnpicklingError) < 0)
+    }
+    if (PyModule_AddObjectRef(m, "UnpicklingError", st->UnpicklingError) < 0) {
         return NULL;
-
+    }
     if (_Pickle_InitState(st) < 0)
         return NULL;