Skip to content
Open
Show file tree
Hide file tree
Changes from 5 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion ftp/src/main/java/ch/cyberduck/core/ftp/FTPClient.java
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,8 @@ protected Socket _openDataConnection_(final String command, final String arg) th
if(null == socket) {
throw new FTPException(this.getReplyCode(), this.getReplyString());
}
return socket;
// Wrap socket to ensure proper TCP shutdown sequence
return new FTPSocket(socket);
}

@Override
Expand Down
305 changes: 305 additions & 0 deletions ftp/src/main/java/ch/cyberduck/core/ftp/FTPSocket.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,305 @@
package ch.cyberduck.core.ftp;

/*
* Copyright (c) 2002-2025 David Kocher. All rights reserved.
* http://cyberduck.ch/
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* Bug fixes, suggestions and comments should be sent to feedback@cyberduck.ch
*/

import org.apache.commons.io.input.ProxyInputStream;
import org.apache.commons.io.output.ProxyOutputStream;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;

import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.Socket;
import java.net.SocketAddress;
import java.net.SocketException;
import java.nio.channels.SocketChannel;
import java.util.concurrent.CompletableFuture;

/**
* Socket wrapper that enforces proper TCP shutdown sequence to prevent
* race conditions due to premature socket closure during FTP data connections.
* <p>
* This fixes issues where:
* - macOS: Intermittent "426 Connection closed; transfer aborted" errors due to socket closure before server ACKs
* - Windows: Intermittent transfer hanging due to socket closure before client sends FIN with last data packet
* <p>
* The proper TCP shutdown sequence is:
* 1. Call shutdownOutput() to send FIN while keeping socket input open for ACKs
* 2. Drain input to wait for ACKs until we receive server FIN
* 3. Close the socket to release resources
*/
public class FTPSocket extends Socket {
private static final Logger log = LogManager.getLogger(FTPSocket.class);

private final Socket delegate;
private InputStream inputStreamWrapper;
private OutputStream outputStreamWrapper;

public FTPSocket(final Socket delegate) {
this.delegate = delegate;
}

@Override
public synchronized void close() throws IOException {
if(delegate.isClosed()) {
log.debug("Socket already closed {}", delegate);
return;
}
try {
if(delegate.isOutputShutdown()) {
log.debug("Socket output already closed {}", delegate);
}
else if(!delegate.isConnected()) {
log.debug("Socket is already disconnected {}", delegate);
}
else {
// Shutdown output to send FIN, but keep socket open to receive ACKs
log.debug("Shutting down output for socket {}", delegate);
delegate.shutdownOutput();

// Wait for server FIN by draining any remaining data
// This ensures all our data packets are ACKed before closing
log.debug("Draining input for socket {}", delegate);
InputStream in = delegate.getInputStream();
byte[] buffer = new byte[8192];
int bytesRead;
// Read until EOF (server closes its side) or timeout
while((bytesRead = in.read(buffer)) != -1) {
log.debug("Drained {} bytes from socket {}", bytesRead, delegate);
}
}
}
finally {
log.debug("Closing socket {}", delegate);
// Work around macOS quirk where Java NIO's SocketDispatcher.close0() has a 1,000ms delay
CompletableFuture.runAsync(() -> {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Running in a background thread and missing any error looks dangerious to me as we will not report any error when closing the stream fails which may indicate that the file is not properly persisted on the remote server.

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I fully agree. I tried really hard to find a workaround, but couldn't come up with one. Having 1 whole second delay between each file upload is really bad when you upload lots of small files, and it causes Cyberduck to be upwards of 10x slower than competitor FTP clients. If you have any suggestion here, I'm more than happy to take it.

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Actually, I don't think this failing specifically could result in data loss. By the time we've reached here we already flushed our data, closed our stream, and waited for the server to close its stream. So, I don't think failing here could ever result in data loss.

The only case of data loss should occur if "Failed to shutdown output for socket" is hit. Maybe we can change that to throw instead of logging and discarding.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You are probably right there is no risk here in the real world. But it should probably not be in the finally block as it does not need to be executed when above code fails with I/O.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is that close quirk on macOS specific to server implementations?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I wouldn't want to include the workaround if it can only be reproduced with a single FTP server implementation.

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

No other FTP client experiences this delay though, so it's clearly on Cyberduck's side. I'm away from home this week but will look more into it next week.

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@dkocher I did a deep dive on this.

  1. I compared Forklift's traffic dump with Cyberduck + this PR, and it is completely identical. Packets are the same and their options (SYN,ACK,FIN) too.
  2. I debugged the JVM native code and traced the 1s delay to macOS close() syscall.
  3. I wrote a script to dump netstat -anl to file and confirmed that in both apps, the socket transitions from SYN_SENT -> ESTABLISHED -> FIN_WAIT_1 -> FIN_WAIT_2 -> TIME_WAIT. Forklift remains in TIME_WAIT for 5s+, while Cyberduck for 1-5s+ depending on SO_LINGER flag. I'm unsure why SO_LINGER(true, 0) results in 1s of TIME_WAIT, as I thought that it should be skipped entirely? This could be a quirk in netstat on macOS though.
  4. I tried to reproduce this issue using a minimal TCP client/server in C, but it does not block there regardless of SO_LINGER.
  5. The only difference between the "1s hang" server and "no hang" server is that the "1s hang" server completes 4-way shutdown of data socket first, and then sends "OK" on control socket, while the "no hang" server sends "OK" on control socket first, and then sends the FIN on data socket. This, and also some packet timing differences.

If the control socket is truly independent of the data socket as I understand it is, then my conclusion is that Socket.close() blocking is purely due to timing. There should be nothing preventing it from happening on any FTP server given that it hits the same timing.

So, unless you have better ideas, I vote to keep the current workaround.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I have just one more suggestion to try out to see what difference it makes with different SO_LINGER options:

  • Are you testing with SSL sockets? We enabled SO_LINGER for this use case if I remember correctly 1
  • What is the behaviour with connection.socket.linger set to false in default.properties or explicitly in DefaultSocketConfigurator
  • Enable SO_LINGER but modify DefaultSocketConfigurator to set with a shorter timeout value of 1. With 0 I see 426 errors for data transfers. 2

Footnotes

  1. 94718d5

  2. 6c2d19f

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I have opened #17562 to make the SO_LINGER timeout configurable.

try {
delegate.close();
}
catch(IOException e) {
log.error("Error closing socket {}: {}", delegate, e.getMessage());
}
});
}
}

@Override
public void connect(SocketAddress endpoint) throws IOException {
delegate.connect(endpoint);
}

@Override
public void connect(SocketAddress endpoint, int timeout) throws IOException {
delegate.connect(endpoint, timeout);
}

@Override
public void bind(SocketAddress bindpoint) throws IOException {
delegate.bind(bindpoint);
}

@Override
public SocketAddress getRemoteSocketAddress() {
return delegate.getRemoteSocketAddress();
}

@Override
public SocketAddress getLocalSocketAddress() {
return delegate.getLocalSocketAddress();
}

@Override
public SocketChannel getChannel() {
return delegate.getChannel();
}

@Override
public synchronized OutputStream getOutputStream() throws IOException {
if(outputStreamWrapper == null) {
outputStreamWrapper = new ProxyOutputStream(delegate.getOutputStream()) {
@Override
public void close() throws IOException {
// We can't call super.close() as it would call delegate.close()
// Therefore, we flush here and close the underlying stream ourselves
try {
super.flush();
}
finally {
FTPSocket.this.close();
}
}
};
}
return outputStreamWrapper;
}

@Override
public synchronized InputStream getInputStream() throws IOException {
if(inputStreamWrapper == null) {
inputStreamWrapper = new ProxyInputStream(delegate.getInputStream()) {
@Override
public void close() throws IOException {
// super.close() will call delegate.close(), so override it with ours instead
FTPSocket.this.close();
}
};
}
return inputStreamWrapper;
}

@Override
public void setTcpNoDelay(boolean on) throws SocketException {
delegate.setTcpNoDelay(on);
}

@Override
public boolean getTcpNoDelay() throws SocketException {
return delegate.getTcpNoDelay();
}

@Override
public void setSoLinger(boolean on, int linger) throws SocketException {
delegate.setSoLinger(on, linger);
}

@Override
public int getSoLinger() throws SocketException {
return delegate.getSoLinger();
}

@Override
public void sendUrgentData(int data) throws IOException {
delegate.sendUrgentData(data);
}

@Override
public void setOOBInline(boolean on) throws SocketException {
delegate.setOOBInline(on);
}

@Override
public boolean getOOBInline() throws SocketException {
return delegate.getOOBInline();
}

@Override
public void setSoTimeout(int timeout) throws SocketException {
delegate.setSoTimeout(timeout);
}

@Override
public int getSoTimeout() throws SocketException {
return delegate.getSoTimeout();
}

@Override
public void setSendBufferSize(int size) throws SocketException {
delegate.setSendBufferSize(size);
}

@Override
public int getSendBufferSize() throws SocketException {
return delegate.getSendBufferSize();
}

@Override
public void setReceiveBufferSize(int size) throws SocketException {
delegate.setReceiveBufferSize(size);
}

@Override
public int getReceiveBufferSize() throws SocketException {
return delegate.getReceiveBufferSize();
}

@Override
public void setKeepAlive(boolean on) throws SocketException {
delegate.setKeepAlive(on);
}

@Override
public boolean getKeepAlive() throws SocketException {
return delegate.getKeepAlive();
}

@Override
public void setTrafficClass(int tc) throws SocketException {
delegate.setTrafficClass(tc);
}

@Override
public int getTrafficClass() throws SocketException {
return delegate.getTrafficClass();
}

@Override
public void setReuseAddress(boolean on) throws SocketException {
delegate.setReuseAddress(on);
}

@Override
public boolean getReuseAddress() throws SocketException {
return delegate.getReuseAddress();
}

@Override
public void shutdownInput() throws IOException {
delegate.shutdownInput();
}

@Override
public void shutdownOutput() throws IOException {
delegate.shutdownOutput();
}

@Override
public boolean isConnected() {
return delegate.isConnected();
}

@Override
public boolean isBound() {
return delegate.isBound();
}

@Override
public boolean isClosed() {
return delegate.isClosed();
}

@Override
public boolean isInputShutdown() {
return delegate.isInputShutdown();
}

@Override
public boolean isOutputShutdown() {
return delegate.isOutputShutdown();
}

@Override
public void setPerformancePreferences(int connectionTime, int latency, int bandwidth) {
delegate.setPerformancePreferences(connectionTime, latency, bandwidth);
}

@Override
public String toString() {
return "FTPSocket{" + delegate + "}";
}
}
Loading