diff --git a/.github/workflows/document.yml b/.github/workflows/document.yml index a2a9f9cc..77d719d1 100644 --- a/.github/workflows/document.yml +++ b/.github/workflows/document.yml @@ -16,7 +16,7 @@ jobs: fail-fast: true matrix: os: [ubuntu-latest] - python-version: ['3.12'] + python-version: ['3.13'] steps: - name: Clone Repository diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 214f3731..f230e330 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -14,7 +14,7 @@ jobs: fail-fast: true matrix: os: [ubuntu-latest] - python-version: ['3.12'] + python-version: ['3.13'] steps: - name: Clone Repository diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b13aa699..590ea12b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -13,7 +13,7 @@ jobs: fail-fast: false matrix: os: [macos-13, macos-latest, ubuntu-22.04, ubuntu-latest, windows-latest] - python-version: ['3.7', '3.8', '3.9', '3.10', '3.11', '3.12'] + python-version: ['3.7', '3.8', '3.9', '3.10', '3.11', '3.12', '3.13'] exclude: - os: macos-latest python-version: 3.7 diff --git a/README.rst b/README.rst index b77e4d1a..9ecb6991 100644 --- a/README.rst +++ b/README.rst @@ -138,6 +138,6 @@ paramiko >= 2.7.0 Supports -------- -Tested on Python 3.6, 3.7, 3.8, 3.9, 3.10, 3.11, 3.12 +Tested on Python 3.6, 3.7, 3.8, 3.9, 3.10, 3.11, 3.12, 3.13 diff --git a/docs/changes.rst b/docs/changes.rst index 8a2b22a1..32b46199 100644 --- a/docs/changes.rst +++ b/docs/changes.rst @@ -1,10 +1,14 @@ -Change Log -========== -1.1.9 (current, released 2025-8-06) + +1.1.10 (current, released 2025-9-19) +----------------------------------- + * fix for channel re-use limitation. + * regression fix for properly closing channel cache sockets. + +1.1.9 (released 2025-8-06) ----------------------------------- - * removing ssh-dss key type as it was deprecated in paramiko in 4.0.0. * adding channel cache to place upper limit on creation overhead. + * removing ssh-dss key type as it was deprecated in paramiko in 4.0.0. 1.1.8 (released 2025-6-13) -------------------------- diff --git a/docs/conf.py b/docs/conf.py index 23e78afa..b3fee7c5 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -54,9 +54,9 @@ # built documents. # # The short X.Y version. -version = '1.1.9' +version = '1.1.10' # The full version, including alpha/beta/rc tags. -release = '1.1.9' +release = '1.1.10' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/docs/cookbook.rst b/docs/cookbook.rst index 11abc7ff..3425f96d 100644 --- a/docs/cookbook.rst +++ b/docs/cookbook.rst @@ -99,7 +99,7 @@ private key or password authentication. # do stuff here Config options always take precedence over parameters if both exist. Keep in -mind there will more than likely be a delta between the security option +mind there it's quite likely there will be a delta between the security option algorithms your verion of SSH supports and those supported by the underlying paramiko dependency. @@ -127,7 +127,7 @@ always attempted unless an alternative is passed. If you wish to disable host key checking, **NOT ADVISED**, you will need to modify the default CnOpts and set the knownhosts to None if no such file exists. You can still modify an existing CnOpts by setting cnopts.hostkeys to None if a default known_hosts -exists or an alternative file was passed when CnOpts was created. +exists or an alternative file was passed when the CnOpts was created. .. code-block:: python @@ -146,7 +146,7 @@ exists or an alternative file was passed when CnOpts was created. with sftpretty.Connection('host', username='me', password='pass', cnopts=cnopts): # do stuff here -To use a completely different known_hosts file, you can override CnOpts looking +To use a completely different known_hosts file, you can override CnOpts search for ``~/.ssh/known_hosts`` by specifying the file when instantiating. .. code-block:: python @@ -201,7 +201,8 @@ Just send the dict into the connection object like so. import sftpretty - cinfo = {'host':'hostname', 'username':'me', 'password':'secret', 'port':2222} + cinfo = {'host': 'hostname', 'username': 'me', + 'password': 'secret', 'port': 2222} with sftpretty.Connection(**cinfo) as sftp: # # ... do sftp operations @@ -219,7 +220,7 @@ the modification times on the local copy match those on the server. # ... sftp.get('myfile', preserve_mtime=True) -Now with the ability to resume a previously started download. Based on local +Resuming a previously initiated download is supported and based on local destination path matching. .. code-block:: python @@ -258,7 +259,7 @@ connections from above still applies. -------------------------------- In addition to the normal paramiko call, you can optionally set the ``preserve_mtime`` parameter to ``True`` and the operation will make sure that -the modification times on the server copy match those on the local. +the modification times on the server copy match those of the local. .. code-block:: python @@ -266,8 +267,8 @@ the modification times on the server copy match those on the local. # preserving modification time sftp.put('myfile', preserve_mtime=True) -Now with the ability to resume a prematurely ended upload. Based on remote -destination path matching. +Resume a prematurely ended upload if desired, still based on destination path +matching. .. code-block:: python diff --git a/docs/index.rst b/docs/index.rst index b138d237..6c4ed36b 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -138,7 +138,7 @@ paramiko >= 2.7.0 Supports -------- -Tested on Python 3.6, 3.7, 3.8, 3.9, 3.10, 3.11, 3.12 +Tested on Python 3.6, 3.7, 3.8, 3.9, 3.10, 3.11, 3.12, 3.13 Contents -------- diff --git a/pyproject.toml b/pyproject.toml index 2d0d62f0..53c98f4a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,6 +22,7 @@ classifiers = [ 'Programming Language :: Python :: 3.10', 'Programming Language :: Python :: 3.11', 'Programming Language :: Python :: 3.12', + 'Programming Language :: Python :: 3.13', 'Programming Language :: Python :: Implementation :: CPython' ] dependencies = [ @@ -46,7 +47,7 @@ keywords = [ name = 'sftpretty' readme = 'README.rst' requires-python = '>=3.6' -version = '1.1.9' +version = '1.1.10' [project.scripts] sftpretty = 'sftpretty:Connection' diff --git a/sftpretty/__init__.py b/sftpretty/__init__.py index 0637efd4..2f0eb6ee 100644 --- a/sftpretty/__init__.py +++ b/sftpretty/__init__.py @@ -14,7 +14,7 @@ from socket import gaierror from stat import S_ISDIR, S_ISREG from tempfile import mkstemp -from threading import get_ident, local as cache +from threading import local as cache from uuid import uuid4 @@ -191,7 +191,8 @@ def __init__(self, host, cnopts=None, default_path=None, password=None, port=22, private_key=None, private_key_pass=None, timeout=None, username=None): self._cache = cache() - self._channels = [] + self._cache.__dict__.setdefault('cwd', default_path) + self._channels = {} self._cnopts = cnopts or CnOpts() self._config = self._cnopts.get_config(host) self._default_path = default_path @@ -294,34 +295,51 @@ def _set_username(self, username): @contextmanager def _sftp_channel(self): '''Establish new SFTP channel.''' - _channel = getattr(self._cache, 'channel', None) + channel = None try: - if _channel is None or _channel.get_channel().closed: - _channel = SFTPClient.from_transport(self._transport) - channel = _channel.get_channel() - channel_name = uuid4().hex - channel.set_name(channel_name) - channel.settimeout(self._timeout) - log.debug(f'Channel Name: [{channel_name}]') - - if self._default_path is not None: - _channel.chdir(drivedrop(self._default_path)) - log.info(('Current Working Directory: ' - f'[{self._default_path}]')) - - self._cache.channel = _channel - self._channels.append(_channel) - log.debug(f'Thread Cached: [{get_ident()}]') - else: - channel = _channel.get_channel() - channel.settimeout(self._timeout) + channel_name, data = next( + (key, value) + for key, value in self._channels.items() + if not value['busy'] + ) + meta = data['meta'] + if not meta.closed: + channel = data['channel'] + self._channels[channel_name]['busy'] = True + log.debug(f'Cached Channel: [{channel_name}]') + except StopIteration: + pass + + if channel is None: + channel = SFTPClient.from_transport(self._transport) + channel_name = uuid4().hex + meta = channel.get_channel() + meta.set_name(channel_name) + log.debug(f'Channel Name: [{channel_name}]') + self._channels[channel_name] = { + 'busy': True, 'channel': channel, 'meta': meta + } + + try: + meta.settimeout(self._timeout) + self._cache.__dict__.setdefault('cwd', self._default_path) - yield _channel + if self._cache.cwd: + try: + channel.chdir(drivedrop(self._cache.cwd)) + log.info(f'Current Working Directory: [{self._cache.cwd}]') + except IOError as err: + log.error(f'Failed Directory Change: [{self._cache.cwd}]') + raise err + + yield channel except Exception as err: - _channel.close() - self._cache.channel = None + channel.close() raise err + finally: + if not meta.closed: + self._channels[channel_name]['busy'] = False def _start_transport(self, host, port): '''Start the transport and set connection options if specified.''' @@ -653,7 +671,7 @@ def get_r(self, remotedir, localdir, callback=None, self.chdir(remotedir) lwd = Path(localdir).absolute().as_posix() - rwd = self._default_path + rwd = self._cache.cwd tree = {} tree[rwd] = [(rwd, lwd)] @@ -1094,7 +1112,8 @@ def chdir(self, remotepath): ''' with self._sftp_channel() as channel: channel.chdir(drivedrop(remotepath)) - self._default_path = channel.normalize('.') + self._cache.cwd = drivedrop(channel.normalize('.')) + self._default_path = self._cache.cwd def chmod(self, remotepath, mode=700): '''Set the permission mode of a remotepath, where mode is an octal. @@ -1137,15 +1156,15 @@ def close(self): '''Terminate transport connection and clean up the bits.''' try: # Close cached channels - for channel in self._channels: - if not channel.closed: - channel.close() + for channel_name, data in self._channels.items(): + if not data['channel'].sock.closed: + data['channel'].close() + # Close the transport. if self._transport and self._transport.is_active(): self._transport.close() - self._cache = cache() - self._channels = [] + self._channels = {} self._transport = None # Clean up any loggers @@ -1247,7 +1266,7 @@ def listdir(self, remotepath='.'): return directory def listdir_attr(self, remotepath='.'): - '''Return a non-sorted list of SFTPAttribute objects for the remote + '''Return a sorted list of SFTPAttribute objects for the remote directory contents. Will not include the special entries '.' and '..'. The returned SFTPAttributes objects will each have an additional field: @@ -1334,7 +1353,7 @@ def normalize(self, remotepath): with self._sftp_channel() as channel: expanded_path = channel.normalize(drivedrop(remotepath)) - return expanded_path + return drivedrop(expanded_path) def open(self, remotefile, bufsize=-1, mode='r'): '''Open a file on the remote server. @@ -1364,7 +1383,7 @@ def readlink(self, remotelink): remotelink = drivedrop(remotelink) link_destination = channel.normalize(channel.readlink(remotelink)) - return link_destination + return drivedrop(link_destination) def remotetree(self, container, remotedir, localdir, recurse=True): '''Recursively map remote directory tree to a dictionary container. @@ -1518,7 +1537,8 @@ def pwd(self): :returns: (str) Current working directory. ''' with self._sftp_channel() as channel: - pwd = channel.normalize('.') + pwd = drivedrop(channel.normalize('.')) + self._default_path = pwd return pwd diff --git a/sftpretty/helpers.py b/sftpretty/helpers.py index f9180b9f..1487ba88 100644 --- a/sftpretty/helpers.py +++ b/sftpretty/helpers.py @@ -17,8 +17,9 @@ def _callback(filename, bytes_so_far, bytes_total, logger=None): def drivedrop(filepath): - if PureWindowsPath(filepath).drive: - filepath = Path('/').joinpath(*Path(filepath).parts[1:]).as_posix() + if filepath: + if PureWindowsPath(filepath).drive: + filepath = Path('/').joinpath(*Path(filepath).parts[1:]).as_posix() return filepath diff --git a/tests/common.py b/tests/common.py index 446e41b4..7317f273 100644 --- a/tests/common.py +++ b/tests/common.py @@ -29,6 +29,7 @@ def conn(sftpsrv): '''return a dictionary holding argument info for the sftpretty client''' cnopts = CnOpts(knownhosts='sftpserver.pub') + cnopts.log_level = 'debug' return {'cnopts': cnopts, 'default_path': '/home/test', 'host': sftpsrv.host, 'port': sftpsrv.port, 'private_key': 'id_sftpretty', 'private_key_pass': PASS, diff --git a/tests/test_cd.py b/tests/test_cd.py index 8f2695a7..10188561 100644 --- a/tests/test_cd.py +++ b/tests/test_cd.py @@ -3,13 +3,13 @@ import pytest from common import conn, VFS -from pathlib import Path +from pathlib import PurePosixPath from sftpretty import Connection def test_cd_none(sftpserver): '''test sftpretty.cd with None''' - pubpath = Path('/home/test').joinpath('pub') + pubpath = PurePosixPath('/home/test').joinpath('pub') with sftpserver.serve_content(VFS): with Connection(**conn(sftpserver)) as sftp: home = sftp.pwd @@ -21,7 +21,7 @@ def test_cd_none(sftpserver): def test_cd_path(sftpserver): '''test sftpretty.cd with a path''' - pubpath = Path('/home/test').joinpath('pub') + pubpath = PurePosixPath('/home/test').joinpath('pub') with sftpserver.serve_content(VFS): with Connection(**conn(sftpserver)) as sftp: home = sftp.pwd @@ -32,7 +32,7 @@ def test_cd_path(sftpserver): def test_cd_nested(sftpserver): '''test nested cd's''' - pubpath = Path('/home/test').joinpath('pub') + pubpath = PurePosixPath('/home/test').joinpath('pub') with sftpserver.serve_content(VFS): with Connection(**conn(sftpserver)) as sftp: home = sftp.pwd diff --git a/tests/test_issue_65.py b/tests/test_issue_65.py index 536ecfe2..46c82fb0 100644 --- a/tests/test_issue_65.py +++ b/tests/test_issue_65.py @@ -2,14 +2,14 @@ location''' from common import conn, VFS -from pathlib import Path +from pathlib import PurePosixPath from sftpretty import Connection def test_issue_65(sftpserver): '''using the .cd() context manager prior to setting a directory via chdir causes an error''' - pubpath = Path('/home/test').joinpath('pub') + pubpath = PurePosixPath('/home/test').joinpath('pub') with sftpserver.serve_content(VFS): cnn = conn(sftpserver) cnn['default_path'] = None @@ -18,4 +18,4 @@ def test_issue_65(sftpserver): with sftp.cd(pubpath.as_posix()): pass - assert sftp.getcwd() == '/' + assert sftp.getcwd() == pubpath.root diff --git a/tests/test_issue_xx.py b/tests/test_issue_xx.py index 8ff00632..122d4cd3 100644 --- a/tests/test_issue_xx.py +++ b/tests/test_issue_xx.py @@ -19,7 +19,7 @@ def test_issue_xx_sftpserver_plugin(sftpserver): with sftp.cd(): sftp.chdir('pub') assert sftp.pwd == testpath.as_posix() - assert home == '/home/test' + assert home == testpath.parent.as_posix() def test_issue_xx_local_sftpserver(lsftp):