Reduce deadlocks by inserting contributors in batches #3036
Conversation
![github-actions[bot]](https://avatars.githubusercontent.com/in/15368?s=60&v=4){kind=small}
| @@ -274,6 +274,17 @@ def facade_bulk_insert_commits(logger, records): | |||
| raise e | |||
|
|
|||
|
|
|||
| def batch_insert_contributors(logger, data: Union[List[dict], dict]) -> Optional[List[dict]]: | |||
There was a problem hiding this comment.
[pylint] reported by reviewdog 🐶
W0621: Redefining name 'logger' from outer scope (line 18) (redefined-outer-name)
| bulk_insert_dicts(logger, batch, Contributor, ['cntrb_id']) | ||
|
|
||
|
|
||
|
|
||
| def bulk_insert_dicts(logger, data: Union[List[dict], dict], table, natural_keys: List[str], return_columns: Optional[List[str]] = None, string_fields: Optional[List[str]] = None, on_conflict_update:bool = True) -> Optional[List[dict]]: |
There was a problem hiding this comment.
[pylint] reported by reviewdog 🐶
W0621: Redefining name 'logger' from outer scope (line 18) (redefined-outer-name)
| @@ -14,7 +14,7 @@ | |||
| from augur.tasks.github.util.util import get_owner_repo | |||
| from augur.tasks.util.worker_util import remove_duplicate_dicts | |||
| from augur.application.db.models import PullRequestEvent, IssueEvent, Contributor, Repo | |||
There was a problem hiding this comment.
[pylint] reported by reviewdog 🐶
W0611: Unused Contributor imported from augur.application.db.models (unused-import)
| @@ -14,7 +14,7 @@ | |||
| from augur.tasks.github.util.util import get_owner_repo | |||
| from augur.tasks.util.worker_util import remove_duplicate_dicts | |||
| from augur.application.db.models import PullRequestEvent, IssueEvent, Contributor, Repo | |||
There was a problem hiding this comment.
[pylint] reported by reviewdog 🐶
W0611: Unused Repo imported from augur.application.db.models (unused-import)
| @@ -14,7 +14,7 @@ | |||
| from augur.tasks.github.util.util import get_owner_repo | |||
| from augur.tasks.util.worker_util import remove_duplicate_dicts | |||
| from augur.application.db.models import PullRequestEvent, IssueEvent, Contributor, Repo | |||
| from augur.application.db.lib import get_repo_by_repo_git, bulk_insert_dicts, get_issues_by_repo_id, get_pull_requests_by_repo_id, update_issue_closed_cntrbs_by_repo_id, get_session, get_engine, get_core_data_last_collected | |||
| from augur.application.db.lib import get_repo_by_repo_git, bulk_insert_dicts, get_issues_by_repo_id, get_pull_requests_by_repo_id, update_issue_closed_cntrbs_by_repo_id, get_session, get_engine, get_core_data_last_collected, batch_insert_contributors | |||
There was a problem hiding this comment.
[pylint] reported by reviewdog 🐶
W0611: Unused get_session imported from augur.application.db.lib (unused-import)
| @@ -4,7 +4,7 @@ | |||
|
|
|||
| from augur.application.db.data_parse import * | |||
| from augur.application.db.session import DatabaseSession | |||
There was a problem hiding this comment.
[pylint] reported by reviewdog 🐶
W0611: Unused DatabaseSession imported from augur.application.db.session (unused-import)
| @@ -4,7 +4,7 @@ | |||
|
|
|||
| from augur.application.db.data_parse import * | |||
| from augur.application.db.session import DatabaseSession | |||
| from augur.application.db.lib import bulk_insert_dicts | |||
| from augur.application.db.lib import bulk_insert_dicts, batch_insert_contributors | |||
| from augur.tasks.github.util.util import add_key_value_pair_to_dicts | |||
| from augur.tasks.util.worker_util import remove_duplicate_dicts | |||
| from augur.application.db.models import PullRequest, PullRequestLabel, PullRequestReviewer, PullRequestMeta, PullRequestAssignee, Contributor | |||
There was a problem hiding this comment.
[pylint] reported by reviewdog 🐶
W0611: Unused Contributor imported from augur.application.db.models (unused-import)
| @@ -11,7 +11,7 @@ | |||
| from augur.application.db.models import PullRequest, Message, PullRequestReview, PullRequestLabel, PullRequestReviewer, PullRequestMeta, PullRequestAssignee, PullRequestReviewMessageRef, Contributor, Repo | |||
| from augur.tasks.github.util.github_task_session import GithubTaskManifest | |||
| from augur.tasks.github.util.github_random_key_auth import GithubRandomKeyAuth | |||
| from augur.application.db.lib import get_session, get_repo_by_repo_git, bulk_insert_dicts, get_pull_request_reviews_by_repo_id | |||
| from augur.application.db.lib import get_session, get_repo_by_repo_git, bulk_insert_dicts, get_pull_request_reviews_by_repo_id, batch_insert_contributors | |||
There was a problem hiding this comment.
[pylint] reported by reviewdog 🐶
W0611: Unused get_session imported from augur.application.db.lib (unused-import)
| @@ -11,7 +11,7 @@ | |||
| from augur.tasks.github.util.util import get_gitlab_repo_identifier, add_key_value_pair_to_dicts | |||
| from augur.application.db.models import Issue, IssueLabel, IssueAssignee, IssueMessageRef, Message, Contributor, Repo | |||
There was a problem hiding this comment.
[pylint] reported by reviewdog 🐶
W0611: Unused Contributor imported from augur.application.db.models (unused-import)
| @@ -8,7 +8,7 @@ | |||
| from augur.application.db.models import PullRequest, PullRequestLabel, PullRequestMeta, PullRequestCommit, PullRequestFile, PullRequestMessageRef, Repo, Message, Contributor, PullRequestAssignee | |||
There was a problem hiding this comment.
[pylint] reported by reviewdog 🐶
W0611: Unused Contributor imported from augur.application.db.models (unused-import)
Signed commits