Fix setSoTimeout() being ignored on Windows

README.md

@@ -525,7 +525,9 @@ wolfSSL JNI/JSSE does not select/poll on a large number of file descriptors
 connections, when using `select()` the `FD_ISSET` and other related macros
 result in undefined behavior when the file descriptor number is larger than
 `FD_SETSIZE` (defaults to 1024 on most systems). For this reason, `poll()` is
-used as the default descriptor monitoring function.
+used as the default descriptor monitoring function. This limit does not apply
+to Windows, where `FD_SETSIZE` is the number of sockets an `fd_set` can hold
+rather than a limit on the socket value.
 
 ### Security Property Support
 

native/com_wolfssl_WolfSSL.c

@@ -3062,7 +3062,7 @@ JNIEXPORT jint JNICALL Java_com_wolfssl_WolfSSL_getErrno
 #ifndef USE_WINDOWS_API
     return errno;
 #else
-    return 0;
+    return WSAGetLastError();
 #endif
 }
 

native/com_wolfssl_WolfSSLSession.c

@@ -88,12 +88,18 @@ static jobject g_crlCbIfaceObj;
  *
  * interruptFds[2] is a pipe() used for non-Windows platforms. This pipe is
  * used to interrupt threads blocked inside select()/poll() when a separate
- * Java thread calls close() on the SSLSocket. */
+ * Java thread calls close() on the SSLSocket. Windows has no pipe() usable
+ * with select(). There, SSLSocket.close() closing the SOCKET wakes threads
+ * blocked in select(), which socketSelect() reports as
+ * WOLFJNI_IO_EVENT_FD_CLOSED, and ioInterrupted makes later socketSelect()
+ * calls return without waiting. */
 typedef struct SSLAppData {
     int threadsInPoll;               /* number of threads in poll/select() */
 #ifndef USE_WINDOWS_API
     int interruptFds[2];             /* pipe for interrupting socketSelect() */
     wolfSSL_Mutex* pollCountLock;    /* lock around threadsInPoll */
+#else
+    volatile int ioInterrupted;      /* set by interruptBlockedIO() */
 #endif
     wolfSSL_Mutex* jniSessLock;      /* WOLFSSL session lock */
     jobject* g_verifySSLCbIfaceObj;  /* Java verify callback [global ref] */
@@ -518,7 +524,8 @@ JNIEXPORT jint JNICALL Java_com_wolfssl_WolfSSLSession_setFd(JNIEnv* jenv,
     jobject impl;
     jobject fdesc;
 #ifdef USE_WINDOWS_API
-    unsigned long blocking = 0;
+    /* FIONBIO enables non-blocking mode when set to non-zero */
+    unsigned long nonBlocking = 1;
 #endif
     WOLFSSL* ssl = (WOLFSSL*)(uintptr_t)sslPtr;
     (void)jcl;
@@ -629,12 +636,17 @@ JNIEXPORT jint JNICALL Java_com_wolfssl_WolfSSLSession_setFd(JNIEnv* jenv,
     /* set socket to non-blocking so we can use select() to detect
      * WANT_READ / WANT_WRITE */
 #ifdef USE_WINDOWS_API
-    ret = ioctlsocket(fd, FIONBIO, &blocking);
-    if (ret == SOCKET_ERROR) {
-        ret = SSL_FAILURE;
-    }
-    else {
-        ret = SSL_SUCCESS;
+    /* Only for Socket (type 1). DatagramSocket stays blocking, since wolfSSL
+     * is not told the socket is non-blocking and relies on a blocking
+     * socket receive timeout to retransmit DTLS handshake messages. */
+    if (type == 1) {
+        ret = ioctlsocket(fd, FIONBIO, &nonBlocking);
+        if (ret == SOCKET_ERROR) {
+            ret = SSL_FAILURE;
+        }
+        else {
+            ret = SSL_SUCCESS;
+        }
     }
 #else
     ret = fcntl(fd, F_SETFL, fcntl(fd, F_GETFL, 0) | O_NONBLOCK);
@@ -841,8 +853,10 @@ enum {
  * ready for read/write, or timeout. Note that we explicitly set the underlying
  * socket descriptor to non-blocking.
  *
- * NOTE: the FD_ISSET macro behavior is undefined if the descriptor value is
- *       less than 0 or greater than or equal to FD_SETSIZE (1024 by default).
+ * NOTE: on POSIX the FD_ISSET macro behavior is undefined if the descriptor
+ *       value is less than 0 or greater than or equal to FD_SETSIZE (1024 by
+ *       default). On Windows FD_SETSIZE is the number of SOCKET slots in an
+ *       fd_set (64 by default), not a limit on the SOCKET value.
  *
  * On a Java Socket, a timeout of 0 is an infinite timeout. Greater than zero
  * is a timeout in milliseconds. Negative timeout is invalid and not supported.
@@ -891,11 +905,27 @@ static int socketSelect(SSLAppData* appData, int sockfd, int timeout_ms, int rx,
         return WOLFJNI_IO_EVENT_ERROR;
     }
 
+#ifdef USE_WINDOWS_API
+    /* Winsock FD_SET stores the SOCKET in the next free slot of fd_array
+     * and ignores it once all FD_SETSIZE slots are used, so SOCKET values
+     * above FD_SETSIZE are valid. Only one SOCKET is placed in each set
+     * here, reject only invalid descriptors. */
+    if (sockfd < 0) {
+        return WOLFJNI_IO_EVENT_ERROR;
+    }
+
+    /* Once SSLSocket.close() has started, return instead of waiting, as the
+     * interrupt pipe does on other platforms. shutdownSSL() still waits. */
+    if ((shutdown == 0) && (appData->ioInterrupted != 0)) {
+        return WOLFJNI_IO_EVENT_FD_CLOSED;
+    }
+#else
     /* Guard against FD_SET overflow. FD_SET behavior is undefined
-     * if sockfd >= FD_SETSIZE (1024 on Linux, 64 on Windows). */
+     * if sockfd >= FD_SETSIZE (1024 by default). */
     if (sockfd < 0 || sockfd >= FD_SETSIZE) {
         return WOLFJNI_IO_EVENT_ERROR;
     }
+#endif
 
 #ifndef USE_WINDOWS_API
     do {
@@ -987,6 +1017,13 @@ static int socketSelect(SSLAppData* appData, int sockfd, int timeout_ms, int rx,
     if ((result == -1) && (errno == EBADF)) {
         return WOLFJNI_IO_EVENT_FD_CLOSED;
     }
+#else
+    /* Winsock select() fails with WSAENOTSOCK if the SOCKET is not valid,
+     * expected cause is the socket being closed concurrently. Report as a
+     * closed fd, matching POSIX EBADF case above. */
+    if ((result == SOCKET_ERROR) && (WSAGetLastError() == WSAENOTSOCK)) {
+        return WOLFJNI_IO_EVENT_FD_CLOSED;
+    }
 #endif
 
     return WOLFJNI_IO_EVENT_FAIL;
@@ -1273,6 +1310,16 @@ static int SSLWriteNonblockingWithSelectPoll(WOLFSSL* ssl, byte* data,
             break;
         }
 
+    #ifdef USE_WINDOWS_API
+        /* Once close() has started, restoreFdBlocking() may have made the
+         * SOCKET blocking, where send() would wait without a timeout */
+        if (appData->ioInterrupted != 0) {
+            wc_UnLockMutex(jniSessLock);
+            ret = WOLFJNI_IO_EVENT_FD_CLOSED;
+            break;
+        }
+    #endif
+
         ret = wolfSSL_write(ssl, data, length);
         err = wolfSSL_get_error(ssl, ret);
 
@@ -1510,6 +1557,16 @@ static int SSLReadNonblockingWithSelectPoll(WOLFSSL* ssl, byte* out,
             break;
         }
 
+    #ifdef USE_WINDOWS_API
+        /* Once close() has started, restoreFdBlocking() may have made the
+         * SOCKET blocking, where recv() would wait without a timeout */
+        if (appData->ioInterrupted != 0) {
+            wc_UnLockMutex(jniSessLock);
+            size = WOLFJNI_IO_EVENT_FD_CLOSED;
+            break;
+        }
+    #endif
+
         size = wolfSSL_read(ssl, out, length);
         err = wolfSSL_get_error(ssl, size);
 
@@ -6902,26 +6959,29 @@ JNIEXPORT jint JNICALL Java_com_wolfssl_WolfSSLSession_useClientSuites
 JNIEXPORT jint JNICALL Java_com_wolfssl_WolfSSLSession_interruptBlockedIO
   (JNIEnv* jenv, jobject jcl, jlong sslPtr)
 {
-#ifndef USE_WINDOWS_API
     SSLAppData* appData = NULL;
     WOLFSSL* ssl = (WOLFSSL*)(uintptr_t)sslPtr;
-#endif
     (void)jenv;
     (void)jcl;
 
-#ifndef USE_WINDOWS_API
     /* get session mutex from SSL app data */
     appData = (SSLAppData*)wolfSSL_get_app_data(ssl);
     if (appData == NULL) {
         return WOLFSSL_FAILURE;
     }
 
+#ifndef USE_WINDOWS_API
     /* signal any blocked threads in select()/poll() to wake up, so we
      * don't have a deadlock when trying to lock jniSessLock next */
     writeToInterruptPipe(appData);
     writeToInterruptPipe(appData);
-
-#endif /* USE_WINDOWS_API */
+#else
+    /* No interrupt pipe on Windows. Make later socketSelect() calls return
+     * instead of waiting, for example the session ticket wait in
+     * get1Session() during close(). Threads already in select() wake when
+     * the SOCKET is closed. */
+    appData->ioInterrupted = 1;
+#endif
 
     return SSL_SUCCESS;
 }
@@ -6944,6 +7004,49 @@ JNIEXPORT jint JNICALL Java_com_wolfssl_WolfSSLSession_getThreadsBlockedInPoll
     return (jint)threadsWaitingInPollSelect(appData);
 }
 
+JNIEXPORT jint JNICALL Java_com_wolfssl_WolfSSLSession_restoreFdBlocking
+  (JNIEnv* jenv, jobject jcl, jlong sslPtr)
+{
+#ifdef USE_WINDOWS_API
+    int fd;
+    int ret = SSL_SUCCESS;
+    unsigned long nonBlocking = 0;
+    SSLAppData* appData = NULL;
+    WOLFSSL* ssl = (WOLFSSL*)(uintptr_t)sslPtr;
+#endif
+    (void)jenv;
+    (void)jcl;
+
+#ifdef USE_WINDOWS_API
+    appData = (SSLAppData*)wolfSSL_get_app_data(ssl);
+    if ((appData == NULL) || (appData->jniSessLock == NULL)) {
+        return SSL_FAILURE;
+    }
+
+    /* Undo FIONBIO set in setFd(). Winsock closesocket() on a non-blocking
+     * SOCKET with SO_LINGER enabled fails with WSAEWOULDBLOCK and leaves the
+     * SOCKET open. Hold jniSessLock so the mode does not change during
+     * another thread's wolfSSL_read()/wolfSSL_write(). Those check
+     * ioInterrupted under the same lock and stop before using the SOCKET
+     * again, so threads still in select() don't resume on a blocking SOCKET. */
+    if (wc_LockMutex(appData->jniSessLock) != 0) {
+        return SSL_FAILURE;
+    }
+    fd = wolfSSL_get_fd(ssl);
+    if ((fd >= 0) &&
+        (ioctlsocket(fd, FIONBIO, &nonBlocking) == SOCKET_ERROR)) {
+        ret = SSL_FAILURE;
+    }
+    wc_UnLockMutex(appData->jniSessLock);
+
+    return ret;
+#else
+    (void)sslPtr;
+
+    return SSL_SUCCESS;
+#endif /* USE_WINDOWS_API */
+}
+
 JNIEXPORT void JNICALL Java_com_wolfssl_WolfSSLSession_setSSLIORecv
     (JNIEnv* jenv, jobject jcl, jlong sslPtr)
 {

src/java/com/wolfssl/WolfSSL.java

@@ -85,13 +85,14 @@ public enum TLS_VERSION {
     public static final int WOLFJNI_IO_EVENT_TIMEOUT = -11;
 
     /**
-     * Socket poll() exceptional error, matches com_wolfssl_WolfSSLSession.c
-     * socketPoll() return value */
+     * Socket select/poll() exceptional error, matches
+     * com_wolfssl_WolfSSLSession.c socketSelect()/socketPoll() return value.
+     */
     public static final int WOLFJNI_IO_EVENT_ERROR = -14;
 
     /**
      * Socket file descriptor closed, matches com_wolfssl_WolfSSLSession.c
-     * socketPoll() return value */
+     * socketSelect() and socketPoll() return value */
     public static final int WOLFJNI_IO_EVENT_FD_CLOSED = -15;
 
     /**
@@ -738,6 +739,10 @@ public WolfSSL() throws WolfSSLException {
                 "Failed to initialize wolfSSL library: " + ret);
         }
 
+        /* Resolve getErrno() early. On Windows the JVM first-call symbol
+         * lookup would overwrite WSAGetLastError() before it is read. */
+        getErrno();
+
         /* Populate NID values from native wolfSSL enums */
         NID_surname = getNID_surname();
         NID_serialNumber = getNID_serialNumber();
@@ -2076,9 +2081,9 @@ public static int cryptoCbUnRegisterDevice(int devId) {
      * error is encountered. This JNI method is a simple getter to retrive
      * the current system errno value.
      *
-     * If on Windows, this will return 0 (no errno support on Windows)
+     * On Windows, this returns the value of WSAGetLastError().
      *
-     * @return System native errno value
+     * @return System native errno value, or WSAGetLastError() on Windows
      */
     public static native int getErrno();
 

src/java/com/wolfssl/WolfSSLSession.java

@@ -760,6 +760,7 @@ private native int useALPNByteArray(long ssl, byte[] protocols,
     private native int useClientSuites(long ssl);
     private native int interruptBlockedIO(long ssl);
     private native int getThreadsBlockedInPoll(long ssl);
+    private native int restoreFdBlocking(long ssl);
     private native int setMTU(long ssl, int mtu);
     private native String stateStringLong(long ssl);
     private native int getMaxOutputSize(long ssl);
@@ -1041,12 +1042,12 @@ private static void throwExceptionFromIOReturnValue(
                 "Native socket timed out during " + nativeFunc);
         }
         else if (ret == WolfSSL.WOLFJNI_IO_EVENT_FD_CLOSED) {
-            throw new SocketException("Socket fd closed during poll(), " +
-                "errno = " + WolfSSL.getErrno());
+            throw new SocketException("Socket fd closed during " +
+                "select/poll(), errno = " + WolfSSL.getErrno());
         }
         else if (ret == WolfSSL.WOLFJNI_IO_EVENT_ERROR) {
-            throw new SocketException("Socket fd poll() exceptional error, " +
-                "errno = " + WolfSSL.getErrno());
+            throw new SocketException("Socket fd select/poll() exceptional " +
+                "error, errno = " + WolfSSL.getErrno());
         }
         else if (ret == WolfSSL.WOLFJNI_IO_EVENT_POLLHUP) {
             throw new SocketException("Socket disconnected during poll(), " +
@@ -5727,6 +5728,12 @@ public void setIOSendByteBuffer(WolfSSLByteBufferIOSendCallback callback)
      *
      * This is used by wolfJSSE when SSLSocket.close() is called, to wake up
      * threads that are blocked in select()/poll().
+     * <p>
+     * On Windows, threads already blocked in select() are not woken by this
+     * call, they wake when the socket is closed. After this call, read()
+     * and write() on this session throw SocketException without using the
+     * socket, and other select() waits return without waiting, except in
+     * shutdownSSL().
      *
      * @return WolfSSL.SSL_SUCCESS on success, negative on error.
      *
@@ -5767,6 +5774,36 @@ public synchronized int getThreadsBlockedInPoll()
         return getThreadsBlockedInPoll(this.sslPtr);
     }
 
+    /**
+     * Put the socket set with setFd() back into blocking mode.
+     *
+     * setFd() puts the socket into non-blocking mode. On Windows,
+     * closesocket() on a non-blocking socket with SO_LINGER enabled fails
+     * and leaves the socket open. Call this after shutdownSSL() and before
+     * the socket is closed. Does nothing on other platforms.
+     *
+     * @return WolfSSL.SSL_SUCCESS on success, WolfSSL.SSL_FAILURE if the
+     *         socket mode could not be changed.
+     *
+     * @throws IllegalStateException WolfSSLSession has been freed
+     */
+    public int restoreFdBlocking() throws IllegalStateException {
+
+        final int ret;
+
+        confirmObjectIsActive();
+
+        synchronized (sslLock) {
+            WolfSSLDebug.log(getClass(), WolfSSLDebug.Component.JNI,
+                WolfSSLDebug.INFO, this.sslPtr,
+                () -> "entered restoreFdBlocking()");
+
+            ret = restoreFdBlocking(this.sslPtr);
+        }
+
+        return ret;
+    }
+
     /**
      * Use SNI name with this session.
      *