Update blocked status documentation (aws#4139)

api/s2n.h

@@ -1835,8 +1835,11 @@ typedef enum {
  * rejects the client certificate, the client may later receive an alert while calling s2n_recv,
  * potentially after already having sent application data with s2n_send.
  *
+ * See the following example for guidance on calling `s2n_negotiate()`:
+ * https://github.com/aws/s2n-tls/blob/main/docs/examples/s2n_negotiate.c
+ *
  * @param conn A pointer to the s2n_connection object
- * @param blocked A pointer which will be set to the blocked status. 
+ * @param blocked A pointer which will be set to the blocked status if an `S2N_ERR_T_BLOCKED` error is returned.
  * @returns S2N_SUCCESS if the handshake completed. S2N_FAILURE if the handshake encountered an error or is blocked.
  */
 S2N_API extern int s2n_negotiate(struct s2n_connection *conn, s2n_blocked_status *blocked);
@@ -1847,24 +1850,15 @@ S2N_API extern int s2n_negotiate(struct s2n_connection *conn, s2n_blocked_status
  *
  * @note Partial writes are possible not just for non-blocking I/O, but also for connections aborted while active. 
  * @note Unlike OpenSSL, repeated calls to s2n_send() should not duplicate the original parameters, but should 
- * update `buf` and `size` per the indication of size written. For example;
- * ```c
- * s2n_blocked_status blocked;
- * int written = 0;
- * char data[10]; 
- * do {
- *     int w = s2n_send(conn, data + written, 10 - written, &blocked);
- *     if (w < 0) {
- *         break;
- *     }
- *     written += w;
- * } while (blocked != S2N_NOT_BLOCKED);
- * ```
+ * update `buf` and `size` per the indication of size written.
+ *
+ * See the following example for guidance on calling `s2n_send()`:
+ * https://github.com/aws/s2n-tls/blob/main/docs/examples/s2n_send.c
  *
  * @param conn A pointer to the s2n_connection object
  * @param buf A pointer to a buffer that s2n will write data from
  * @param size The size of buf
- * @param blocked A pointer which will be set to the blocked status, as in s2n_negotiate()
+ * @param blocked A pointer which will be set to the blocked status if an `S2N_ERR_T_BLOCKED` error is returned.
  * @returns The number of bytes written, and may indicate a partial write
  */
 S2N_API extern ssize_t s2n_send(struct s2n_connection *conn, const void *buf, ssize_t size, s2n_blocked_status *blocked);
@@ -1877,7 +1871,7 @@ S2N_API extern ssize_t s2n_send(struct s2n_connection *conn, const void *buf, ss
  * @param conn A pointer to the s2n_connection object
  * @param bufs A pointer to a vector of buffers that s2n will write data from.
  * @param count The number of buffers in `bufs`
- * @param blocked A pointer which will be set to the blocked status, as in s2n_negotiate()
+ * @param blocked A pointer which will be set to the blocked status if an `S2N_ERR_T_BLOCKED` error is returned.
  * @returns The number of bytes written, and may indicate a partial write. 
  */
 S2N_API extern ssize_t s2n_sendv(struct s2n_connection *conn, const struct iovec *bufs, ssize_t count, s2n_blocked_status *blocked);
@@ -1887,29 +1881,16 @@ S2N_API extern ssize_t s2n_sendv(struct s2n_connection *conn, const struct iovec
  *
  * @note Partial writes are possible not just for non-blocking I/O, but also for connections aborted while active.
  *
- * @note Unlike OpenSSL, repeated calls to s2n_sendv_with_offset() should not duplicate the original parameters, but should update `bufs` and `count` per the indication of size written. For example;
- * 
- * ```c
- * s2n_blocked_status blocked;
- * int written = 0;
- * char data[10]; 
- * struct iovec iov[1];
- * iov[0].iov_base = data;
- * iov[0].iov_len = 10;
- * do {
- *     int w = s2n_sendv_with_offset(conn, iov, 1, written, &blocked);
- *     if (w < 0) {
- *         break;
- *     }
- *     written += w;
- * } while (blocked != S2N_NOT_BLOCKED);
- * ```
+ * @note Unlike OpenSSL, repeated calls to s2n_sendv_with_offset() should not duplicate the original parameters, but should update `bufs` and `count` per the indication of size written.
+ *
+ * See the following example for guidance on calling `s2n_sendv_with_offset()`:
+ * https://github.com/aws/s2n-tls/blob/main/docs/examples/s2n_send.c
  *
  * @param conn A pointer to the s2n_connection object
  * @param bufs A pointer to a vector of buffers that s2n will write data from.
  * @param count The number of buffers in `bufs`
  * @param offs The write cursor offset. This should be updated as data is written. See the example code.
- * @param blocked A pointer which will be set to the blocked status, as in s2n_negotiate()
+ * @param blocked A pointer which will be set to the blocked status if an `S2N_ERR_T_BLOCKED` error is returned.
  * @returns The number of bytes written, and may indicate a partial write. 
  */
 S2N_API extern ssize_t s2n_sendv_with_offset(struct s2n_connection *conn, const struct iovec *bufs, ssize_t count, ssize_t offs, s2n_blocked_status *blocked);
@@ -1918,24 +1899,15 @@ S2N_API extern ssize_t s2n_sendv_with_offset(struct s2n_connection *conn, const
  * Decrypts and reads **size* to `buf` data from the associated
  * connection. 
  * 
- * @note Unlike OpenSSL, repeated calls to `s2n_recv` should not duplicate the original parameters, but should update `buf` and `size` per the indication of size read. For example;
- * ```c
- * s2n_blocked_status blocked;
- * int bytes_read = 0;
- * char data[10];
- * do {
- *     int r = s2n_recv(conn, data + bytes_read, 10 - bytes_read, &blocked);
- *     if (r < 0) {
- *         break;
- *     }
- *     bytes_read += r;
- * } while (blocked != S2N_NOT_BLOCKED);
- * ```
+ * @note Unlike OpenSSL, repeated calls to `s2n_recv` should not duplicate the original parameters, but should update `buf` and `size` per the indication of size read.
+ *
+ * See the following example for guidance on calling `s2n_recv()`:
+ * https://github.com/aws/s2n-tls/blob/main/docs/examples/s2n_recv.c
  *
  * @param conn A pointer to the s2n_connection object
  * @param buf A pointer to a buffer that s2n will place read data into.
  * @param size Size of `buf`
- * @param blocked A pointer which will be set to the blocked status, as in s2n_negotiate()
+ * @param blocked A pointer which will be set to the blocked status if an `S2N_ERR_T_BLOCKED` error is returned.
  * @returns number of bytes read. 0 if the connection was shutdown by peer.
  */
 S2N_API extern ssize_t s2n_recv(struct s2n_connection *conn, void *buf, ssize_t size, s2n_blocked_status *blocked);
@@ -2009,7 +1981,7 @@ S2N_API extern int s2n_connection_free(struct s2n_connection *conn);
  * * The s2n_connection handle can be freed via s2n_connection_free() or reused via s2n_connection_wipe()
  *
  * @param conn A pointer to the s2n_connection object
- * @param blocked A pointer which will be set to the blocked status, as in s2n_negotiate()
+ * @param blocked A pointer which will be set to the blocked status if an `S2N_ERR_T_BLOCKED` error is returned.
  * @returns S2N_SUCCESS on success. S2N_FAILURE on failure
  */
 S2N_API extern int s2n_shutdown(struct s2n_connection *conn, s2n_blocked_status *blocked);
@@ -2037,7 +2009,7 @@ S2N_API extern int s2n_shutdown(struct s2n_connection *conn, s2n_blocked_status
  * the read side of the underlying transport.
  *
  * @param conn A pointer to the s2n_connection object
- * @param blocked A pointer which will be set to the blocked status, as in s2n_negotiate()
+ * @param blocked A pointer which will be set to the blocked status if an `S2N_ERR_T_BLOCKED` error is returned.
  * @returns S2N_SUCCESS on success. S2N_FAILURE on failure
  */
 S2N_API extern int s2n_shutdown_send(struct s2n_connection *conn, s2n_blocked_status *blocked);

docs/USAGE-GUIDE.md

@@ -329,11 +329,16 @@ how to handle C signals, or simply ignore the SIGPIPE signal by calling
 s2n-tls supports both blocking and non-blocking I/O.
 * In blocking mode, each s2n-tls I/O function will not return until it has completed
 the requested IO operation.
-* In non-blocking mode, an s2n-tls I/O function may return while there is
-still I/O pending. If **S2N_FAILURE** is returned, `s2n_error_get_type()`
-will report **S2N_ERR_T_BLOCKED**. The s2n_blocked_status parameter will be
-set to indicate which IO direction blocked. s2n-tls I/O functions should be called
-repeatedly until the **blocked** parameter is **S2N_NOT_BLOCKED**.
+* In non-blocking mode, s2n-tls I/O functions will immediately return, even if the socket couldn't
+send or receive all the requested data. In this case, the I/O function will return `S2N_FAILURE`,
+and `s2n_error_get_type()` will return `S2N_ERR_T_BLOCKED`. The I/O operation will have to be
+called again in order to send or receive the remaining requested data.
+
+Some s2n-tls I/O functions take a `blocked` argument. If an I/O function returns an
+`S2N_ERR_T_BLOCKED` error, the `blocked` argument will be set to a `s2n_blocked_status` value,
+indicating what s2n-tls is currently blocked on. Note that unless an I/O function returns
+`S2N_FAILURE` with an `S2N_ERR_T_BLOCKED` error, the `blocked` argument is meaningless, and should
+not be used in any application logic.
 
 Servers in particular usually prefer non-blocking mode. In blocking mode, a single connection
 blocks the thread while waiting for more IO. In non-blocking mode, multiple connections
@@ -401,9 +406,15 @@ provided application data. s2n-tls breaks the application data into fixed-sized
 records before encryption, and calls write for each record.
 [See the record size documentation for how record size may impact performance](https://github.com/aws/s2n-tls/blob/main/docs/USAGE-GUIDE.md#record-sizes).
 
-To ensure that all data passed to `s2n_send()` is successfully sent, either call
-`s2n_send()` until its s2n_blocked_status parameter is **S2N_NOT_BLOCKED** or
-until the return values across all calls have added up to the length of the data.
+In non-blocking mode, `s2n_send()` will send data from the provided buffer and return the number of
+bytes sent, as long as the socket was able to send at least 1 byte. If no bytes could be sent on the
+socket, `s2n_send()` will return `S2N_FAILURE`, and `s2n_error_get_type()` will return
+`S2N_ERR_T_BLOCKED`. To ensure that all the provided data gets sent, applications should continue
+calling `s2n_send()` until the return values across all calls have added up to the length of the
+data, or until `s2n_send()` returns an `S2N_ERR_T_BLOCKED` error. After an `S2N_ERR_T_BLOCKED`
+error is returned, applications should call `s2n_send()` again only after the socket is
+able to send more data. This can be determined by using methods like
+[`poll`](https://linux.die.net/man/2/poll) or [`select`](https://linux.die.net/man/2/select).
 
 Unlike OpenSSL, repeated calls to `s2n_send()` should not duplicate the original
 parameters, but should update the inputs per the indication of size written.
@@ -426,11 +437,18 @@ the application-provided output buffer. It returns the number of bytes read, and
 may indicate a partial read even if blocking IO is used.
 It returns "0" to indicate that the peer has shutdown the connection.
 
-By default, `s2n_recv()` will return after reading a single TLS record.
-To instead read until the provided output buffer is full, call `s2n_recv()` until
-its s2n_blocked_status parameter is **S2N_NOT_BLOCKED**. Alternatively, use
-`s2n_config_set_recv_multi_record()` to configure s2n-tls to read multiple
-TLS records until the provided output buffer is full.
+By default, `s2n_recv()` will return after reading a single TLS record. `s2n_recv()` can be called
+repeatedly to read multiple records. To allow `s2n_recv()` to read multiple records with a single
+call, use `s2n_config_set_recv_multi_record()`.
+
+In non-blocking mode, `s2n_recv()` will read data into the provided buffer and return the number of
+bytes read, as long as at least 1 byte was read from the socket. If no bytes could be read from the
+socket, `s2n_recv()` will return `S2N_FAILURE`, and `s2n_error_get_type()` will return
+`S2N_ERR_T_BLOCKED`. To ensure that all data on the socket is properly received, applications
+should continue calling `s2n_recv()` until it returns an `S2N_ERR_T_BLOCKED` error. After an
+`S2N_ERR_T_BLOCKED` error is returned, applications should call `s2n_recv()` again only after the
+socket has received more data. This can be determined by using methods like 
+[`poll`](https://linux.die.net/man/2/poll) or [`select`](https://linux.die.net/man/2/select).
 
 Unlike OpenSSL, repeated calls to `s2n_recv()` should not duplicate the original parameters,
 but should update the inputs per the indication of size read.

docs/examples/s2n_recv.c

@@ -21,11 +21,26 @@ int s2n_example_recv(struct s2n_connection *conn, uint8_t *buffer, size_t buffer
     int bytes_read = 0;
     while (bytes_read < buffer_size) {
         int r = s2n_recv(conn, buffer + bytes_read, buffer_size - bytes_read, &blocked);
-        if (r == 0) {
-            break;
-        } else if (r > 0) {
+        if (r > 0) {
+            /* `r` bytes were successfully received. Update `bytes_read` so the next `s2n_recv()`
+             * call writes into the buffer with the correct offset.
+             */
             bytes_read += r;
-        } else if (s2n_error_get_type(s2n_errno) != S2N_ERR_T_BLOCKED) {
+        } else if (r == 0) {
+            /* The connection was closed. No more data will be received, so bail out of the loop. */
+            break;
+        } else if (s2n_error_get_type(s2n_errno) == S2N_ERR_T_BLOCKED) {
+            /* The return of `s2n_recv()` indicates an error. If this error is `S2N_ERR_T_BLOCKED`,
+             * the socket is blocked waiting to receive more data from the peer.
+             *
+             * In a typical non-blocking environment, `poll()` or `select()` would be used re-enter
+             * `s2n_example_recv()` and call `s2n_recv()` again after the socket has more data
+             * available. But this example just busy-waits, so we immediately call `s2n_recv()`
+             * again.
+             */
+            continue;
+        } else {
+            /* `s2n_recv()` encountered an irrecoverable error. Log the error and return. */
             fprintf(stderr, "Error: %s. %s\n", s2n_strerror(s2n_errno, NULL), s2n_strerror_debug(s2n_errno, NULL));
             return -1;
         }

docs/examples/s2n_send.c

@@ -22,8 +22,22 @@ int s2n_example_send(struct s2n_connection *conn, uint8_t *data, size_t data_siz
     while (bytes_written < data_size) {
         int w = s2n_send(conn, data + bytes_written, data_size - bytes_written, &blocked);
         if (w >= 0) {
+            /* `w` bytes were successfully sent. Update `bytes_written` so the next call to
+             * `s2n_send()` will send data from the correct offset in the buffer.
+             */
             bytes_written += w;
-        } else if (s2n_error_get_type(s2n_errno) != S2N_ERR_T_BLOCKED) {
+        } else if (s2n_error_get_type(s2n_errno) == S2N_ERR_T_BLOCKED) {
+            /* The return of `s2n_send()` indicates an error. If this error is `S2N_ERR_T_BLOCKED`,
+             * the socket is blocked waiting to send more data.
+             *
+             * In a typical non-blocking environment, `poll()` or `select()` would be used re-enter
+             * `s2n_example_send()` and call `s2n_send()` again after the socket is ready to send
+             * more data. But this example just busy-waits, so we immediately call `s2n_send()`
+             * again.
+             */
+            continue;
+        } else {
+            /* `s2n_send()` encountered an irrecoverable error. Log the error and return. */
             fprintf(stderr, "Error: %s. %s\n", s2n_strerror(s2n_errno, NULL), s2n_strerror_debug(s2n_errno, NULL));
             return -1;
         }