INFO:mozsvc.metrics:{"code": 401, "request_time": 0.00031304359436035156, "remoteAddressChain": ["1.2.3.4", "172.17.0.1"], "agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:83.0) Gecko/20100101 Firefox/83.0", "path": "https://example.com/token/1.0/sync/1.5", "method": "GET"}
DEBUG:urllib3.connectionpool:Resetting dropped connection: oauth.accounts.firefox.com
DEBUG:urllib3.connectionpool:https://oauth.accounts.firefox.com:443 "GET /v1/jwks HTTP/1.1" 200 463
DEBUG:urllib3.connectionpool:https://oauth.accounts.firefox.com:443 "POST /v1/verify HTTP/1.1" 200 163
ERROR:mozsvc:Uncaught exception while processing request:
GET https://example.com/token/1.0/sync/1.5
Uncaught exception:
  File "/usr/local/lib/python2.7/site-packages/mozsvc/tweens.py", line 59, in log_uncaught_exceptions_tween
    return handler(request)
  File "/usr/local/lib/python2.7/site-packages/mozsvc/tweens.py", line 26, in catch_backend_errors_tween
    return handler(request)
  File "/usr/local/lib/python2.7/site-packages/pyramid/tweens.py", line 21, in excview_tween
    response = handler(request)
  File "/usr/local/lib/python2.7/site-packages/pyramid/router.py", line 163, in handle_request
    response = view_callable(context, request)
  File "/usr/local/lib/python2.7/site-packages/pyramid/config/views.py", line 596, in __call__
    return view(context, request)
  File "/usr/local/lib/python2.7/site-packages/pyramid/config/views.py", line 329, in attr_view
    return view(context, request)
  File "/usr/local/lib/python2.7/site-packages/pyramid/config/views.py", line 305, in predicate_wrapper
    return view(context, request)
  File "/usr/local/lib/python2.7/site-packages/pyramid/config/views.py", line 355, in rendered_view
    result = view(context, request)
  File "/usr/local/lib/python2.7/site-packages/pyramid/config/views.py", line 501, in _requestonly_view
    response = view(request)
  File "/usr/local/lib/python2.7/site-packages/cornice/service.py", line 514, in wrapper
    response = view_(request)
  File "/usr/local/lib/python2.7/site-packages/tokenserver/views.py", line 370, in return_token
    user = backend.get_user(service, email)
  File "/app/syncserver/staticnode.py", line 135, in get_user
    res = self._engine.execute(_GET_USER_RECORDS, **params)
  File "/usr/local/lib/python2.7/site-packages/sqlalchemy/engine/base.py", line 2166, in execute
    return connection.execute(statement, *multiparams, **params)
  File "/usr/local/lib/python2.7/site-packages/sqlalchemy/engine/base.py", line 988, in execute
    return meth(self, multiparams, params)
  File "/usr/local/lib/python2.7/site-packages/sqlalchemy/sql/elements.py", line 287, in _execute_on_connection
    return connection._execute_clauseelement(self, multiparams, params)
  File "/usr/local/lib/python2.7/site-packages/sqlalchemy/engine/base.py", line 1107, in _execute_clauseelement
    distilled_params,
  File "/usr/local/lib/python2.7/site-packages/sqlalchemy/engine/base.py", line 1248, in _execute_context
    e, statement, parameters, cursor, context
  File "/usr/local/lib/python2.7/site-packages/sqlalchemy/engine/base.py", line 1466, in _handle_dbapi_exception
    util.raise_from_cause(sqlalchemy_exception, exc_info)
  File "/usr/local/lib/python2.7/site-packages/sqlalchemy/util/compat.py", line 383, in raise_from_cause
    reraise(type(exception), exception, tb=exc_tb, cause=cause)
  File "/usr/local/lib/python2.7/site-packages/sqlalchemy/engine/base.py", line 1244, in _execute_context
    cursor, statement, parameters, context
  File "/usr/local/lib/python2.7/site-packages/sqlalchemy/engine/default.py", line 552, in do_execute
    cursor.execute(statement, parameters)
  File "/usr/local/lib/python2.7/site-packages/pymysql/cursors.py", line 170, in execute
    result = self._query(query)
  File "/usr/local/lib/python2.7/site-packages/pymysql/cursors.py", line 328, in _query
    conn.query(q)
  File "/usr/local/lib/python2.7/site-packages/pymysql/connections.py", line 517, in query
    self._affected_rows = self._read_query_result(unbuffered=unbuffered)
  File "/usr/local/lib/python2.7/site-packages/pymysql/connections.py", line 732, in _read_query_result
    result.read()
  File "/usr/local/lib/python2.7/site-packages/pymysql/connections.py", line 1075, in read
    first_packet = self.connection._read_packet()
  File "/usr/local/lib/python2.7/site-packages/pymysql/connections.py", line 657, in _read_packet
    packet_header = self._read_bytes(4)
  File "/usr/local/lib/python2.7/site-packages/pymysql/connections.py", line 707, in _read_bytes
    CR.CR_SERVER_LOST, "Lost connection to MySQL server during query")
<class 'sqlalchemy.exc.OperationalError'>
OperationalError("(pymysql.err.OperationalError) (2013, 'Lost connection to MySQL server during query')",)

INFO:mozsvc.metrics:{"tokenserver.backend.get_user": 0.0009849071502685547, "code": 999, "uid": "2da9f13cb6dd194fc86db7485dd0e7e4520e2f71dc1237906413c8a36bfcf562584", "metrics_uid": "2da9f13cb1236dd194fc86db7485dd0e7e4", "metrics_device_id": "918552d397f712181232d8538d362db09f3", "remoteAddressChain": ["1.2.3.4", "172.17.0.1"], "agent": "Mozilla/5.0 (Android 9; Mobile; rv:83.0) Gecko/83.0 Firefox/83.0", "token.oauth.verify_success": 1, "email": "02f831231dca13543b0a799af12ed9bc422@api.accounts.firefox.com", "path": "https://example.com/token/1.0/sync/1.5", "method": "GET", "tokenserver.oauth.verify": 0.8217251300811768, "request_time": 0.8240809440612793}
[2020-12-11 21:36:47 +0000] [16] [ERROR] Error handling request /token/1.0/sync/1.5
Traceback (most recent call last):
  File "/usr/local/lib/python2.7/site-packages/gunicorn/workers/sync.py", line 135, in handle
    self.handle_request(listener, req, client, addr)
  File "/usr/local/lib/python2.7/site-packages/gunicorn/workers/sync.py", line 176, in handle_request
    respiter = self.wsgi(environ, resp.start_response)
  File "/usr/local/lib/python2.7/site-packages/pyramid/router.py", line 242, in __call__
    response = self.invoke_subrequest(request, use_tweens=True)
  File "/usr/local/lib/python2.7/site-packages/pyramid/router.py", line 217, in invoke_subrequest
    response = handle_request(request)
  File "/usr/local/lib/python2.7/site-packages/tokenserver/tweens.py", line 15, in set_x_timestamp_header_tween
    response = handler(request)
  File "/usr/local/lib/python2.7/site-packages/syncstorage/tweens.py", line 128, in convert_non_json_responses_tween
    response = handler(request)
  File "/usr/local/lib/python2.7/site-packages/syncstorage/tweens.py", line 104, in convert_cornice_errors_to_respcodes_tween
    response = handler(request)
  File "/usr/local/lib/python2.7/site-packages/syncstorage/tweens.py", line 58, in set_default_accept_header_tween
    return handler(request)
  File "/usr/local/lib/python2.7/site-packages/syncstorage/tweens.py", line 30, in set_x_timestamp_header_tween
    response = handler(request)
  File "/usr/local/lib/python2.7/site-packages/mozsvc/tweens.py", line 94, in fuzz_backoff_headers_tween
    response = handler(request)
  File "/usr/local/lib/python2.7/site-packages/mozsvc/tweens.py", line 59, in log_uncaught_exceptions_tween
    return handler(request)
  File "/usr/local/lib/python2.7/site-packages/mozsvc/tweens.py", line 26, in catch_backend_errors_tween
    return handler(request)
  File "/usr/local/lib/python2.7/site-packages/pyramid/tweens.py", line 21, in excview_tween
    response = handler(request)
  File "/usr/local/lib/python2.7/site-packages/pyramid/router.py", line 163, in handle_request
    response = view_callable(context, request)
  File "/usr/local/lib/python2.7/site-packages/pyramid/config/views.py", line 596, in __call__
    return view(context, request)
  File "/usr/local/lib/python2.7/site-packages/pyramid/config/views.py", line 329, in attr_view
    return view(context, request)
  File "/usr/local/lib/python2.7/site-packages/pyramid/config/views.py", line 305, in predicate_wrapper
    return view(context, request)
  File "/usr/local/lib/python2.7/site-packages/pyramid/config/views.py", line 355, in rendered_view
    result = view(context, request)
  File "/usr/local/lib/python2.7/site-packages/pyramid/config/views.py", line 501, in _requestonly_view
    response = view(request)
  File "/usr/local/lib/python2.7/site-packages/cornice/service.py", line 514, in wrapper
    response = view_(request)
  File "/usr/local/lib/python2.7/site-packages/tokenserver/views.py", line 370, in return_token
    user = backend.get_user(service, email)
  File "/app/syncserver/staticnode.py", line 135, in get_user
    res = self._engine.execute(_GET_USER_RECORDS, **params)
  File "/usr/local/lib/python2.7/site-packages/sqlalchemy/engine/base.py", line 2166, in execute
    return connection.execute(statement, *multiparams, **params)
  File "/usr/local/lib/python2.7/site-packages/sqlalchemy/engine/base.py", line 988, in execute
    return meth(self, multiparams, params)
  File "/usr/local/lib/python2.7/site-packages/sqlalchemy/sql/elements.py", line 287, in _execute_on_connection
    return connection._execute_clauseelement(self, multiparams, params)
  File "/usr/local/lib/python2.7/site-packages/sqlalchemy/engine/base.py", line 1107, in _execute_clauseelement
    distilled_params,
  File "/usr/local/lib/python2.7/site-packages/sqlalchemy/engine/base.py", line 1248, in _execute_context
    e, statement, parameters, cursor, context
  File "/usr/local/lib/python2.7/site-packages/sqlalchemy/engine/base.py", line 1466, in _handle_dbapi_exception
    util.raise_from_cause(sqlalchemy_exception, exc_info)
  File "/usr/local/lib/python2.7/site-packages/sqlalchemy/util/compat.py", line 383, in raise_from_cause
    reraise(type(exception), exception, tb=exc_tb, cause=cause)
  File "/usr/local/lib/python2.7/site-packages/sqlalchemy/engine/base.py", line 1244, in _execute_context
    cursor, statement, parameters, context
  File "/usr/local/lib/python2.7/site-packages/sqlalchemy/engine/default.py", line 552, in do_execute
    cursor.execute(statement, parameters)
  File "/usr/local/lib/python2.7/site-packages/pymysql/cursors.py", line 170, in execute
    result = self._query(query)
  File "/usr/local/lib/python2.7/site-packages/pymysql/cursors.py", line 328, in _query
    conn.query(q)
  File "/usr/local/lib/python2.7/site-packages/pymysql/connections.py", line 517, in query
    self._affected_rows = self._read_query_result(unbuffered=unbuffered)
  File "/usr/local/lib/python2.7/site-packages/pymysql/connections.py", line 732, in _read_query_result
    result.read()
  File "/usr/local/lib/python2.7/site-packages/pymysql/connections.py", line 1075, in read
    first_packet = self.connection._read_packet()
  File "/usr/local/lib/python2.7/site-packages/pymysql/connections.py", line 657, in _read_packet
    packet_header = self._read_bytes(4)
  File "/usr/local/lib/python2.7/site-packages/pymysql/connections.py", line 707, in _read_bytes
    CR.CR_SERVER_LOST, "Lost connection to MySQL server during query")
OperationalError: (pymysql.err.OperationalError) (2013, 'Lost connection to MySQL server during query')
[SQL: select
    uid, generation, client_state, created_at, replaced_at,
    keys_changed_at, node
from
    users
where
    email = %(email)s
and
    service = %(service)s
order by
    created_at desc, uid desc
limit
    20
]
[parameters: {'email': u'02f831dca11233543b0a799af12ed9bc422@api.accounts.firefox.com', 'service': u'sync-1.5'}]
(Background on this error at: http://sqlalche.me/e/e3q8)