Skip to content

s3 plugin: saving sometimes fails on minio with custom region #275

@bpedersen2

Description

@bpedersen2

Due to missing region setting option, the s3 plugin fails on minio (self-hosted) storage with custom region settings.

It seems to be dependent on the size ( >1MB fails regularly)

Patch is coming.

typical traceback:

025-07-14 14:10:47,001  08df70cf76a34214  1264    indico.flask - ERROR errors.py:113 -- Could not save "...: SSL validation failed for https://.... EOF occurred in violation of protocol (_ssl.c:2427)

Traceback (most recent call last):
  File "/opt/indico/.venv/lib/python3.12/site-packages/urllib3/connectionpool.py", line 716, in urlopen
    httplib_response = self._make_request(
                       ^^^^^^^^^^^^^^^^^^^
  File "/opt/indico/.venv/lib/python3.12/site-packages/urllib3/connectionpool.py", line 414, in _make_request
    conn.request_chunked(method, url, **httplib_request_kw)
  File "/opt/indico/.venv/lib/python3.12/site-packages/urllib3/connection.py", line 280, in request_chunked
    self.send(to_send)
  File "/opt/indico/.venv/lib/python3.12/site-packages/botocore/awsrequest.py", line 223, in send
    return super().send(str)
           ^^^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.12/http/client.py", line 1057, in send
    self.sock.sendall(data)
  File "/usr/local/lib/python3.12/ssl.py", line 1210, in sendall
    v = self.send(byte_view[count:])
        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.12/ssl.py", line 1179, in send
    return self._sslobj.write(data)
           ^^^^^^^^^^^^^^^^^^^^^^^^
ssl.SSLEOFError: EOF occurred in violation of protocol (_ssl.c:2427)

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/opt/indico/.venv/lib/python3.12/site-packages/botocore/httpsession.py", line 464, in send
    urllib_response = conn.urlopen(
                      ^^^^^^^^^^^^^
  File "/opt/indico/.venv/lib/python3.12/site-packages/urllib3/connectionpool.py", line 802, in urlopen
    retries = retries.increment(
              ^^^^^^^^^^^^^^^^^^
  File "/opt/indico/.venv/lib/python3.12/site-packages/urllib3/util/retry.py", line 527, in increment
    raise six.reraise(type(error), error, _stacktrace)
          ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/opt/indico/.venv/lib/python3.12/site-packages/urllib3/packages/six.py", line 769, in reraise
    raise value.with_traceback(tb)
  File "/opt/indico/.venv/lib/python3.12/site-packages/urllib3/connectionpool.py", line 716, in urlopen
    httplib_response = self._make_request(
                       ^^^^^^^^^^^^^^^^^^^
  File "/opt/indico/.venv/lib/python3.12/site-packages/urllib3/connectionpool.py", line 414, in _make_request
    conn.request_chunked(method, url, **httplib_request_kw)
  File "/opt/indico/.venv/lib/python3.12/site-packages/urllib3/connection.py", line 280, in request_chunked
    self.send(to_send)
  File "/opt/indico/.venv/lib/python3.12/site-packages/botocore/awsrequest.py", line 223, in send
    return super().send(str)
           ^^^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.12/http/client.py", line 1057, in send
    self.sock.sendall(data)
  File "/usr/local/lib/python3.12/ssl.py", line 1210, in sendall
    v = self.send(byte_view[count:])
        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.12/ssl.py", line 1179, in send
    return self._sslobj.write(data)
           ^^^^^^^^^^^^^^^^^^^^^^^^
urllib3.exceptions.SSLError: EOF occurred in violation of protocol (_ssl.c:2427)

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/opt/indico/.venv/lib/python3.12/site-packages/indico_storage_s3/storage.py", line 210, in save
    checksum = self._save(bucket, name, content_type, fileobj) if not dry_run else None
               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/opt/indico/.venv/lib/python3.12/site-packages/indico_storage_s3/storage.py", line 124, in _save
    self.client.upload_fileobj(Fileobj=fileobj, Bucket=bucket, Key=name, ExtraArgs=metadata, Config=transfer_config)
  File "/opt/indico/.venv/lib/python3.12/site-packages/botocore/context.py", line 123, in wrapper
    return func(*args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^
  File "/opt/indico/.venv/lib/python3.12/site-packages/boto3/s3/inject.py", line 675, in upload_fileobj
    return future.result()
           ^^^^^^^^^^^^^^^
  File "/opt/indico/.venv/lib/python3.12/site-packages/s3transfer/futures.py", line 111, in result
    return self._coordinator.result()
           ^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/opt/indico/.venv/lib/python3.12/site-packages/s3transfer/futures.py", line 272, in result
    raise self._exception
  File "/opt/indico/.venv/lib/python3.12/site-packages/s3transfer/tasks.py", line 142, in __call__
    return self._execute_main(kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/opt/indico/.venv/lib/python3.12/site-packages/s3transfer/tasks.py", line 165, in _execute_main
    return_value = self._main(**kwargs)
                   ^^^^^^^^^^^^^^^^^^^^
  File "/opt/indico/.venv/lib/python3.12/site-packages/s3transfer/upload.py", line 796, in _main
    client.put_object(Bucket=bucket, Key=key, Body=body, **extra_args)
  File "/opt/indico/.venv/lib/python3.12/site-packages/botocore/client.py", line 595, in _api_call
    return self._make_api_call(operation_name, kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/opt/indico/.venv/lib/python3.12/site-packages/botocore/context.py", line 123, in wrapper
    return func(*args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^
  File "/opt/indico/.venv/lib/python3.12/site-packages/botocore/client.py", line 1040, in _make_api_call
    http, parsed_response = self._make_request(
                            ^^^^^^^^^^^^^^^^^^^
  File "/opt/indico/.venv/lib/python3.12/site-packages/botocore/client.py", line 1064, in _make_request
    return self._endpoint.make_request(operation_model, request_dict)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/opt/indico/.venv/lib/python3.12/site-packages/botocore/endpoint.py", line 119, in make_request
    return self._send_request(request_dict, operation_model)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/opt/indico/.venv/lib/python3.12/site-packages/botocore/endpoint.py", line 200, in _send_request
    while self._needs_retry(
          ^^^^^^^^^^^^^^^^^^
  File "/opt/indico/.venv/lib/python3.12/site-packages/botocore/endpoint.py", line 360, in _needs_retry
    responses = self._event_emitter.emit(
                ^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/opt/indico/.venv/lib/python3.12/site-packages/botocore/hooks.py", line 412, in emit
    return self._emitter.emit(aliased_event_name, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/opt/indico/.venv/lib/python3.12/site-packages/botocore/hooks.py", line 256, in emit
    return self._emit(event_name, kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/opt/indico/.venv/lib/python3.12/site-packages/botocore/hooks.py", line 239, in _emit
    response = handler(**kwargs)
               ^^^^^^^^^^^^^^^^^
  File "/opt/indico/.venv/lib/python3.12/site-packages/botocore/retryhandler.py", line 207, in __call__
    if self._checker(**checker_kwargs):
       ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/opt/indico/.venv/lib/python3.12/site-packages/botocore/retryhandler.py", line 284, in __call__
    should_retry = self._should_retry(
                   ^^^^^^^^^^^^^^^^^^^
  File "/opt/indico/.venv/lib/python3.12/site-packages/botocore/retryhandler.py", line 320, in _should_retry
    return self._checker(attempt_number, response, caught_exception)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/opt/indico/.venv/lib/python3.12/site-packages/botocore/retryhandler.py", line 363, in __call__
    checker_response = checker(
                       ^^^^^^^^
  File "/opt/indico/.venv/lib/python3.12/site-packages/botocore/retryhandler.py", line 247, in __call__
    return self._check_caught_exception(
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/opt/indico/.venv/lib/python3.12/site-packages/botocore/retryhandler.py", line 416, in _check_caught_exception
    raise caught_exception
  File "/opt/indico/.venv/lib/python3.12/site-packages/botocore/endpoint.py", line 279, in _do_get_response
    http_response = self._send(request)
                    ^^^^^^^^^^^^^^^^^^^
  File "/opt/indico/.venv/lib/python3.12/site-packages/botocore/endpoint.py", line 383, in _send
    return self.http_session.send(request)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/opt/indico/.venv/lib/python3.12/site-packages/botocore/httpsession.py", line 491, in send
    raise SSLError(endpoint_url=request.url, error=e)
botocore.exceptions.SSLError: SSL validation failed for https://.... occurred in violation of protocol (_ssl.c:2427)

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
  File "/opt/indico/.venv/lib/python3.12/site-packages/flask/app.py", line 917, in full_dispatch_request
    rv = self.dispatch_request()
         ^^^^^^^^^^^^^^^^^^^^^^^
  File "/opt/indico/.venv/lib/python3.12/site-packages/flask/app.py", line 902, in dispatch_request
    return self.ensure_sync(self.view_functions[rule.endpoint])(**view_args)  # type: ignore[no-any-return]
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/opt/indico/.venv/lib/python3.12/site-packages/indico/modules/designer/blueprint.py", line 29, in view_func
    return categ_view(**kwargs) if kwargs['object_type'] == 'category' else event_view(**kwargs)
                                                                            ^^^^^^^^^^^^^^^^^^^^
  File "/opt/indico/.venv/lib/python3.12/site-packages/indico/web/flask/util.py", line 80, in wrapper
    return obj().process()
           ^^^^^^^^^^^^^^^
  File "/opt/indico/.venv/lib/python3.12/site-packages/indico/web/rh.py", line 312, in process
    res = self._do_process()
          ^^^^^^^^^^^^^^^^^^
  File "/opt/indico/.venv/lib/python3.12/site-packages/indico/web/rh.py", line 280, in _do_process
    rv = self._process()
         ^^^^^^^^^^^^^^^
  File "/opt/indico/.venv/lib/python3.12/site-packages/indico/modules/designer/controllers.py", line 250, in _process
    return self.clone_template()
           ^^^^^^^^^^^^^^^^^^^^^
  File "/opt/indico/.venv/lib/python3.12/site-packages/indico/modules/designer/controllers.py", line 215, in clone_template
    new_image.save(f)
  File "/opt/indico/.venv/lib/python3.12/site-packages/indico/core/storage/models.py", line 206, in save
    self.storage_file_id, self.md5 = self.storage.save(path, self.content_type, self.filename, data)
                                     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/opt/indico/.venv/lib/python3.12/site-packages/indico_storage_s3/storage.py", line 213, in save
    raise StorageError(f'Could not save "{name}": {exc}') from exc
indico.core.storage.backend.StorageError: Could not save "....: SSL validation failed for .... EOF occurred in violation of protocol (_ssl.c:2427)

Metadata

Metadata

Assignees

No one assigned

    Labels

    No labels
    No labels

    Type

    No type

    Projects

    No projects

    Milestone

    No milestone

    Relationships

    None yet

    Development

    No branches or pull requests

    Issue actions