Hi all. I was experimenting with py-ansible-libssh, but I didn't went to far as currently the port is affected by multiple issues:
- Fix pushing large files over SCP https://github.com/ansible/pylibssh/pull/661 - Prevent SEGFAULTs on consecutive exec_command() invocations https://github.com/ansible/pylibssh/pull/658 - Fix large SFTP reads https://github.com/ansible/pylibssh/pull/638 - sftp.get downloads only the last chunk of file https://github.com/ansible/pylibssh/issues/341 - SIGSEGV when running ssh_channel.exec_command() over different channels https://github.com/ansible/pylibssh/issues/657 Finally all related pull request are merged to upstream's develop branch. I am backporting here commits which I was affected by and I cannot reproduce any issues I've seen before. Well, except significantly slower SFTP trasnfers vs SCP, but at least they work now. I would like to see this in, as currently for me the port as is, in the repo is not really usable. Index: Makefile =================================================================== RCS file: /cvs/ports/sysutils/py-ansible-libssh/Makefile,v diff -u -p -u -r1.3 Makefile --- Makefile 26 Dec 2024 16:15:11 -0000 1.3 +++ Makefile 27 Mar 2025 20:52:34 -0000 @@ -3,7 +3,7 @@ COMMENT = Python bindings for libssh sp DISTNAME = ansible-pylibssh-${MODPY_DISTV} PKGNAME = py-ansible-libssh-${MODPY_DISTV} CATEGORIES = sysutils -REVISION = 0 +REVISION = 1 MAINTAINER = Denis Fondras <de...@openbsd.org> @@ -16,6 +16,8 @@ MODULES = lang/python MODPY_PI = Yes # uses own in-tree backend which wraps setuptools MODPY_PYBUILD = setuptools_scm + +DEBUG_PACKAGES = ${BUILD_PACKAGES} CFLAGS += -I${LOCALBASE}/include BUILD_DEPENDS = sysutils/py-expandvars \ Index: patches/patch-src_pylibsshext_channel_pxd =================================================================== RCS file: patches/patch-src_pylibsshext_channel_pxd diff -N patches/patch-src_pylibsshext_channel_pxd --- /dev/null 1 Jan 1970 00:00:00 -0000 +++ patches/patch-src_pylibsshext_channel_pxd 27 Mar 2025 20:52:34 -0000 @@ -0,0 +1,13 @@ +git -P diff 041d447a94e9dc4aead5aaed1296bdb7dffc5017^ HEAD -- src/ + +Index: src/pylibsshext/channel.pxd +--- src/pylibsshext/channel.pxd.orig ++++ src/pylibsshext/channel.pxd +@@ -23,3 +23,7 @@ cdef class Channel: + cdef _session + cdef libssh.ssh_channel _libssh_channel + cdef libssh.ssh_session _libssh_session ++ ++cdef class ChannelCallback: ++ cdef callbacks.ssh_channel_callbacks_struct callback ++ cdef _userdata Index: patches/patch-src_pylibsshext_channel_pyx =================================================================== RCS file: patches/patch-src_pylibsshext_channel_pyx diff -N patches/patch-src_pylibsshext_channel_pyx --- /dev/null 1 Jan 1970 00:00:00 -0000 +++ patches/patch-src_pylibsshext_channel_pyx 27 Mar 2025 20:52:34 -0000 @@ -0,0 +1,53 @@ +git -P diff 041d447a94e9dc4aead5aaed1296bdb7dffc5017^ HEAD -- src/ + +Index: src/pylibsshext/channel.pyx +--- src/pylibsshext/channel.pyx.orig ++++ src/pylibsshext/channel.pyx +@@ -45,6 +45,16 @@ cdef int _process_outputs(libssh.ssh_session session, + result.stdout += data_b + return len + ++cdef class ChannelCallback: ++ def __cinit__(self): ++ memset(&self.callback, 0, sizeof(self.callback)) ++ callbacks.ssh_callbacks_init(&self.callback) ++ self.callback.channel_data_function = <callbacks.ssh_channel_data_callback>&_process_outputs ++ ++ def set_user_data(self, userdata): ++ self._userdata = userdata ++ self.callback.userdata = <void *>self._userdata ++ + cdef class Channel: + def __cinit__(self, session): + self._session = session +@@ -159,19 +169,23 @@ cdef class Channel: + libssh.ssh_channel_free(channel) + raise LibsshChannelException("Failed to open_session: [{0}]".format(rc)) + ++ result = CompletedProcess(args=command, returncode=-1, stdout=b'', stderr=b'') ++ ++ cb = ChannelCallback() ++ cb.set_user_data(result) ++ callbacks.ssh_set_channel_callbacks(channel, &cb.callback) ++ # keep the callback around in the session object to avoid use after free ++ self._session.push_callback(cb) ++ + rc = libssh.ssh_channel_request_exec(channel, command.encode("utf-8")) + if rc != libssh.SSH_OK: + libssh.ssh_channel_close(channel) + libssh.ssh_channel_free(channel) + raise LibsshChannelException("Failed to execute command [{0}]: [{1}]".format(command, rc)) +- result = CompletedProcess(args=command, returncode=-1, stdout=b'', stderr=b'') + +- cdef callbacks.ssh_channel_callbacks_struct cb +- memset(&cb, 0, sizeof(cb)) +- cb.channel_data_function = <callbacks.ssh_channel_data_callback>&_process_outputs +- cb.userdata = <void *>result +- callbacks.ssh_callbacks_init(&cb) +- callbacks.ssh_set_channel_callbacks(channel, &cb) ++ # wait before remote writes all data before closing the channel ++ while not libssh.ssh_channel_is_eof(channel): ++ libssh.ssh_channel_poll(channel, 0) + + libssh.ssh_channel_send_eof(channel) + result.returncode = libssh.ssh_channel_get_exit_status(channel) Index: patches/patch-src_pylibsshext_includes_libssh_pxd =================================================================== RCS file: patches/patch-src_pylibsshext_includes_libssh_pxd diff -N patches/patch-src_pylibsshext_includes_libssh_pxd --- /dev/null 1 Jan 1970 00:00:00 -0000 +++ patches/patch-src_pylibsshext_includes_libssh_pxd 27 Mar 2025 20:52:34 -0000 @@ -0,0 +1,17 @@ +$ git -P log --oneline e9935cbcf62f7d2f83b9234f6db4e41a3ca480cb^..HEAD -- src/pylibsshext/includes/libssh.pxd +265cae1 sftp: Increase SFTP chunk size to specification-recommended 32kB + +Index: src/pylibsshext/includes/libssh.pxd +--- src/pylibsshext/includes/libssh.pxd.orig ++++ src/pylibsshext/includes/libssh.pxd +@@ -27,6 +27,10 @@ cdef extern from "libssh/libssh.h" nogil: + pass + ctypedef ssh_session_struct* ssh_session + ++ cdef struct ssh_string_struct: ++ pass ++ ctypedef ssh_string_struct* ssh_string ++ + cdef struct ssh_key_struct: + pass + ctypedef ssh_key_struct* ssh_key Index: patches/patch-src_pylibsshext_includes_sftp_pxd =================================================================== RCS file: patches/patch-src_pylibsshext_includes_sftp_pxd diff -N patches/patch-src_pylibsshext_includes_sftp_pxd --- /dev/null 1 Jan 1970 00:00:00 -0000 +++ patches/patch-src_pylibsshext_includes_sftp_pxd 27 Mar 2025 20:52:34 -0000 @@ -0,0 +1,61 @@ +$ git -P log --oneline e9935cbcf62f7d2f83b9234f6db4e41a3ca480cb^..HEAD -- src/pylibsshext/includes/sftp.pxd +265cae1 sftp: Increase SFTP chunk size to specification-recommended 32kB + +Index: src/pylibsshext/includes/sftp.pxd +--- src/pylibsshext/includes/sftp.pxd.orig ++++ src/pylibsshext/includes/sftp.pxd +@@ -17,9 +17,11 @@ + # + from posix.types cimport mode_t + +-from pylibsshext.includes.libssh cimport ssh_channel, ssh_session ++from libc cimport stdint + ++from pylibsshext.includes.libssh cimport ssh_channel, ssh_session, ssh_string + ++ + cdef extern from "libssh/sftp.h" nogil: + + struct sftp_session_struct: +@@ -30,6 +32,31 @@ cdef extern from "libssh/sftp.h" nogil: + pass + ctypedef sftp_file_struct * sftp_file + ++ struct sftp_attributes_struct: ++ char *name ++ char *longname ++ stdint.uint32_t flags ++ stdint.uint8_t type ++ stdint.uint64_t size ++ stdint.uint32_t uid ++ stdint.uint32_t gid ++ char *owner ++ char *group ++ stdint.uint32_t permissions ++ stdint.uint64_t atime64 ++ stdint.uint32_t atime ++ stdint.uint32_t atime_nseconds ++ stdint.uint64_t createtime ++ stdint.uint32_t createtime_nseconds ++ stdint.uint64_t mtime64 ++ stdint.uint32_t mtime ++ stdint.uint32_t mtime_nseconds ++ ssh_string acl ++ stdint.uint32_t extended_count ++ ssh_string extended_type ++ ssh_string extended_data ++ ctypedef sftp_attributes_struct * sftp_attributes ++ + cdef int SSH_FX_OK + cdef int SSH_FX_EOF + cdef int SSH_FX_NO_SUCH_FILE +@@ -54,6 +81,9 @@ cdef extern from "libssh/sftp.h" nogil: + ssize_t sftp_write(sftp_file file, const void *buf, size_t count) + ssize_t sftp_read(sftp_file file, const void *buf, size_t count) + int sftp_get_error(sftp_session sftp) ++ ++ sftp_attributes sftp_stat(sftp_session session, const char *path) ++ + + cdef extern from "sys/stat.h" nogil: + cdef int S_IRWXU Index: patches/patch-src_pylibsshext_scp_pyx =================================================================== RCS file: patches/patch-src_pylibsshext_scp_pyx diff -N patches/patch-src_pylibsshext_scp_pyx --- /dev/null 1 Jan 1970 00:00:00 -0000 +++ patches/patch-src_pylibsshext_scp_pyx 27 Mar 2025 20:52:34 -0000 @@ -0,0 +1,48 @@ +$ git -P log --oneline a5292bdce6948e5eda91d6ff7008c7cc0f912054^..HEAD -- src/pylibsshext/scp.pyx +63ab9a4 scp: Make the chunk size constant more readable +a5292bd scp: Fix pushing large files + +https://github.com/ansible/pylibssh/pull/661 + +Index: src/pylibsshext/scp.pyx +--- src/pylibsshext/scp.pyx.orig ++++ src/pylibsshext/scp.pyx +@@ -24,7 +24,7 @@ from pylibsshext.errors cimport LibsshSCPException + from pylibsshext.session cimport get_libssh_session + + +-SCP_MAX_CHUNK = 65536 ++SCP_MAX_CHUNK = 65_536 # 64kB + + + cdef class SCP: +@@ -74,15 +74,25 @@ cdef class SCP: + ) + + try: ++ # Read buffer ++ read_buffer_size = min(file_size, SCP_MAX_CHUNK) ++ + # Begin to send to the file + rc = libssh.ssh_scp_push_file(scp, filename_b, file_size, file_mode) + if rc != libssh.SSH_OK: + raise LibsshSCPException("Can't open remote file: %s" % self._get_ssh_error_str()) + +- # Write to the open file +- rc = libssh.ssh_scp_write(scp, PyBytes_AS_STRING(f.read()), file_size) +- if rc != libssh.SSH_OK: +- raise LibsshSCPException("Can't write to remote file: %s" % self._get_ssh_error_str()) ++ remaining_bytes_to_read = file_size ++ while remaining_bytes_to_read > 0: ++ # Read the chunk from local file ++ read_bytes = min(remaining_bytes_to_read, read_buffer_size) ++ read_buffer = f.read(read_bytes) ++ ++ # Write to the open file ++ rc = libssh.ssh_scp_write(scp, PyBytes_AS_STRING(read_buffer), read_bytes) ++ if rc != libssh.SSH_OK: ++ raise LibsshSCPException("Can't write to remote file: %s" % self._get_ssh_error_str()) ++ remaining_bytes_to_read -= read_bytes + finally: + libssh.ssh_scp_close(scp) + libssh.ssh_scp_free(scp) Index: patches/patch-src_pylibsshext_session_pxd =================================================================== RCS file: patches/patch-src_pylibsshext_session_pxd diff -N patches/patch-src_pylibsshext_session_pxd --- /dev/null 1 Jan 1970 00:00:00 -0000 +++ patches/patch-src_pylibsshext_session_pxd 27 Mar 2025 20:52:34 -0000 @@ -0,0 +1,12 @@ +git -P diff 041d447a94e9dc4aead5aaed1296bdb7dffc5017^ HEAD -- src/ + +Index: src/pylibsshext/session.pxd +--- src/pylibsshext/session.pxd.orig ++++ src/pylibsshext/session.pxd +@@ -26,5 +26,6 @@ cdef class Session: + cdef _hash_py + cdef _fingerprint_py + cdef _keytype_py ++ cdef _channel_callbacks + + cdef libssh.ssh_session get_libssh_session(Session session) Index: patches/patch-src_pylibsshext_session_pyx =================================================================== RCS file: patches/patch-src_pylibsshext_session_pyx diff -N patches/patch-src_pylibsshext_session_pyx --- /dev/null 1 Jan 1970 00:00:00 -0000 +++ patches/patch-src_pylibsshext_session_pyx 27 Mar 2025 20:52:34 -0000 @@ -0,0 +1,26 @@ +git -P diff 041d447a94e9dc4aead5aaed1296bdb7dffc5017^ HEAD -- src/ + +Index: src/pylibsshext/session.pyx +--- src/pylibsshext/session.pyx.orig ++++ src/pylibsshext/session.pyx +@@ -107,6 +107,10 @@ cdef class Session(object): + self._hash_py = None + self._fingerprint_py = None + self._keytype_py = None ++ # Due to delayed freeing of channels, some older libssh versions might expect ++ # the callbacks to be around even after we free the underlying channels so ++ # we should free them only when we terminate the session. ++ self._channel_callbacks = [] + + def __cinit__(self, host=None, **kwargs): + self._libssh_session = libssh.ssh_new() +@@ -122,6 +126,9 @@ cdef class Session(object): + libssh.ssh_disconnect(self._libssh_session) + libssh.ssh_free(self._libssh_session) + self._libssh_session = NULL ++ ++ def push_callback(self, callback): ++ self._channel_callbacks.append(callback) + + @property + def port(self): Index: patches/patch-src_pylibsshext_sftp_pyx =================================================================== RCS file: patches/patch-src_pylibsshext_sftp_pyx diff -N patches/patch-src_pylibsshext_sftp_pyx --- /dev/null 1 Jan 1970 00:00:00 -0000 +++ patches/patch-src_pylibsshext_sftp_pyx 27 Mar 2025 20:52:34 -0000 @@ -0,0 +1,106 @@ +$ git -P log --oneline e9935cbcf62f7d2f83b9234f6db4e41a3ca480cb^..HEAD -- src/pylibsshext/sftp.pyx +265cae1 sftp: Increase SFTP chunk size to specification-recommended 32kB +d3678e7 sftp: Fix overwriting existing files while downloading +e9935cb sftp: Fix downloading files + +Index: src/pylibsshext/sftp.pyx +--- src/pylibsshext/sftp.pyx.orig ++++ src/pylibsshext/sftp.pyx +@@ -18,11 +18,15 @@ + from posix.fcntl cimport O_CREAT, O_RDONLY, O_TRUNC, O_WRONLY + + from cpython.bytes cimport PyBytes_AS_STRING ++from cpython.mem cimport PyMem_Free, PyMem_Malloc + + from pylibsshext.errors cimport LibsshSFTPException + from pylibsshext.session cimport get_libssh_session + + ++SFTP_MAX_CHUNK = 32_768 # 32kB ++ ++ + MSG_MAP = { + sftp.SSH_FX_OK: "No error", + sftp.SSH_FX_EOF: "End-of-file encountered", +@@ -63,7 +67,7 @@ cdef class SFTP: + rf = sftp.sftp_open(self._libssh_sftp_session, remote_file_b, O_WRONLY | O_CREAT | O_TRUNC, sftp.S_IRWXU) + if rf is NULL: + raise LibsshSFTPException("Opening remote file [%s] for write failed with error [%s]" % (remote_file, self._get_sftp_error_str())) +- buffer = f.read(1024) ++ buffer = f.read(SFTP_MAX_CHUNK) + + while buffer != b"": + length = len(buffer) +@@ -76,38 +80,54 @@ cdef class SFTP: + self._get_sftp_error_str(), + ) + ) +- buffer = f.read(1024) ++ buffer = f.read(SFTP_MAX_CHUNK) + sftp.sftp_close(rf) + + def get(self, remote_file, local_file): + cdef sftp.sftp_file rf +- cdef char read_buffer[1024] ++ cdef char *read_buffer = NULL ++ cdef sftp.sftp_attributes attrs + + remote_file_b = remote_file + if isinstance(remote_file_b, unicode): + remote_file_b = remote_file.encode("utf-8") + ++ attrs = sftp.sftp_stat(self._libssh_sftp_session, remote_file_b) ++ if attrs is NULL: ++ raise LibsshSFTPException("Failed to stat the remote file [%s]. Error: [%s]" ++ % (remote_file, self._get_sftp_error_str())) ++ file_size = attrs.size ++ + rf = sftp.sftp_open(self._libssh_sftp_session, remote_file_b, O_RDONLY, sftp.S_IRWXU) + if rf is NULL: + raise LibsshSFTPException("Opening remote file [%s] for read failed with error [%s]" % (remote_file, self._get_sftp_error_str())) + +- while True: +- file_data = sftp.sftp_read(rf, <void *>read_buffer, sizeof(char) * 1024) +- if file_data == 0: +- break +- elif file_data < 0: +- sftp.sftp_close(rf) +- raise LibsshSFTPException("Reading data from remote file [%s] failed with error [%s]" +- % (remote_file, self._get_sftp_error_str())) ++ try: ++ with open(local_file, 'wb') as f: ++ buffer_size = min(SFTP_MAX_CHUNK, file_size) ++ read_buffer = <char *>PyMem_Malloc(buffer_size) ++ if read_buffer is NULL: ++ raise LibsshSFTPException("Memory allocation error") + +- with open(local_file, 'wb+') as f: +- bytes_written = f.write(read_buffer[:file_data]) +- if bytes_written and file_data != bytes_written: +- sftp.sftp_close(rf) +- raise LibsshSFTPException("Number of bytes [%s] read from remote file [%s]" +- " does not match number of bytes [%s] written to local file [%s]" +- " due to error [%s]" +- % (file_data, remote_file, bytes_written, local_file, self._get_sftp_error_str())) ++ while True: ++ file_data = sftp.sftp_read(rf, <void *>read_buffer, sizeof(char) * buffer_size) ++ if file_data == 0: ++ break ++ elif file_data < 0: ++ sftp.sftp_close(rf) ++ raise LibsshSFTPException("Reading data from remote file [%s] failed with error [%s]" ++ % (remote_file, self._get_sftp_error_str())) ++ ++ bytes_written = f.write(read_buffer[:file_data]) ++ if bytes_written and file_data != bytes_written: ++ sftp.sftp_close(rf) ++ raise LibsshSFTPException("Number of bytes [%s] read from remote file [%s]" ++ " does not match number of bytes [%s] written to local file [%s]" ++ " due to error [%s]" ++ % (file_data, remote_file, bytes_written, local_file, self._get_sftp_error_str())) ++ finally: ++ if read_buffer is not NULL: ++ PyMem_Free(read_buffer) + sftp.sftp_close(rf) + + def close(self): -- Regards, Mikolaj
