Allow limiting the total amount of heap allocations for buffering

This commit introduces a compile time constant MBEDTLS_SSL_DTLS_MAX_BUFFERING
to mbedtls/config.h which allows the user to control the cumulative size of
all heap buffer allocated for the purpose of reassembling and buffering
handshake messages.

It is put to use by introducing a new field `total_bytes_buffered` to
the buffering substructure of `mbedtls_ssl_handshake_params` that keeps
track of the total size of heap allocated buffers for the purpose of
reassembly and buffering at any time. It is increased whenever a handshake
message is buffered or prepared for reassembly, and decreased when a
buffered or fully reassembled message is copied into the input buffer
and passed to the handshake logic layer.

This commit does not yet include future epoch record buffering into
account; this will be done in a subsequent commit.

Also, it is now conceivable that the reassembly of the next expected
handshake message fails because too much buffering space has already
been used up for future messages. This case currently leads to an
error, but instead, the stack should get rid of buffered messages
to be able to buffer the next one. This will need to be implemented
in one of the next commits.

include/mbedtls/config.h

@@ -3010,6 +3010,14 @@
  */
 //#define MBEDTLS_SSL_OUT_CONTENT_LEN             16384
 
+/** \def MBEDTLS_SSL_DTLS_MAX_BUFFERING
+ *
+ * Maximum number of heap-allocated bytes for the purpose of
+ * DTLS handshake message reassembly and future message buffering.
+ *
+ */
+//#define MBEDTLS_SSL_DTLS_MAX_BUFFERING             ( 2 * 16384 )
+
 //#define MBEDTLS_SSL_DEFAULT_TICKET_LIFETIME     86400 /**< Lifetime of session tickets (if enabled) */
 //#define MBEDTLS_PSK_MAX_LEN               32 /**< Max size of TLS pre-shared keys, in bytes (default 256 bits) */
 //#define MBEDTLS_SSL_COOKIE_TIMEOUT        60 /**< Default expiration delay of DTLS cookies, in seconds if HAVE_TIME, or in number of cookies issued */

include/mbedtls/ssl.h

@@ -243,6 +243,10 @@
 #define MBEDTLS_SSL_OUT_CONTENT_LEN MBEDTLS_SSL_MAX_CONTENT_LEN
 #endif
 
+#if !defined(MBEDTLS_SSL_DTLS_MAX_BUFFERING)
+#define MBEDTLS_SSL_DTLS_MAX_BUFFERING ( 2 * MBEDTLS_SSL_IN_CONTENT_LEN )
+#endif
+
 /* \} name SECTION: Module settings */
 
 /*

include/mbedtls/ssl_internal.h

@@ -311,6 +311,9 @@ struct mbedtls_ssl_handshake_params
 
     struct
     {
+        size_t total_bytes_buffered; /*!< Cumulative size of heap allocated
+                                      *   buffers used for message buffering. */
+
         uint8_t seen_ccs;               /*!< Indicates if a CCS message has
                                          *   been seen in the current flight. */
 
@@ -320,6 +323,7 @@ struct mbedtls_ssl_handshake_params
             uint8_t is_fragmented : 1;
             uint8_t is_complete   : 1;
             unsigned char *data;
+            size_t data_len;
         } hs[MBEDTLS_SSL_MAX_BUFFERED_HS];
 
         struct

library/ssl_tls.c

@@ -3665,7 +3665,10 @@ void mbedtls_ssl_update_handshake_status( mbedtls_ssl_context *ssl )
         /* Free first entry */
         hs_buf = &hs->buffering.hs[0];
         if( hs_buf->is_valid )
+        {
+            hs->buffering.total_bytes_buffered -= hs_buf->data_len;
             mbedtls_free( hs_buf->data );
+        }
 
         /* Shift all other entries */
         for( offset = 0; offset + 1 < MBEDTLS_SSL_MAX_BUFFERED_HS;
@@ -4506,18 +4509,49 @@ static int ssl_buffer_message( mbedtls_ssl_context *ssl )
                     goto exit;
                 }
 
+                /* Check if we have enough space to buffer the message. */
+                if( hs->buffering.total_bytes_buffered >
+                    MBEDTLS_SSL_DTLS_MAX_BUFFERING )
+                {
+                    MBEDTLS_SSL_DEBUG_MSG( 1, ( "should never happen" ) );
+                    return( MBEDTLS_ERR_SSL_INTERNAL_ERROR );
+                }
+
                 reassembly_buf_sz = ssl_get_reassembly_buffer_size( msg_len,
                                                        hs_buf->is_fragmented );
+
+                if( reassembly_buf_sz > ( MBEDTLS_SSL_DTLS_MAX_BUFFERING -
+                                          hs->buffering.total_bytes_buffered ) )
+                {
+                    if( recv_msg_seq_offset > 0 )
+                    {
+                        /* If we can't buffer a future message because
+                         * of space limitations -- ignore. */
+                        MBEDTLS_SSL_DEBUG_MSG( 2, ( "Buffering of future message of size %u would exceed the compile-time limit %u (already %u bytes buffered) -- ignore\n",
+                             (unsigned) msg_len, MBEDTLS_SSL_DTLS_MAX_BUFFERING,
+                             (unsigned) hs->buffering.total_bytes_buffered ) );
+                        goto exit;
+                    }
+
+                    /* TODO: Remove future messages in the attempt to make
+                     *       space for the current one. */
+                    MBEDTLS_SSL_DEBUG_MSG( 2, ( "Reassembly of next message of size %u would exceed the compile-time limit %u (already %u bytes buffered) -- fail\n",
+                             (unsigned) msg_len, MBEDTLS_SSL_DTLS_MAX_BUFFERING,
+                             (unsigned) hs->buffering.total_bytes_buffered ) );
+                    ret = MBEDTLS_ERR_SSL_ALLOC_FAILED;
+                    goto exit;
+                }
+
+                MBEDTLS_SSL_DEBUG_MSG( 2, ( "initialize reassembly, total length = %d",
+                                            msg_len ) );
+
                 hs_buf->data = mbedtls_calloc( 1, reassembly_buf_sz );
                 if( hs_buf->data == NULL )
                 {
-                    /* If we run out of RAM trying to buffer a *future*
+                    ret = MBEDTLS_ERR_SSL_ALLOC_FAILED;
-                     * message, simply ignore instead of failing. */
-                    MBEDTLS_SSL_DEBUG_MSG( 2, ( "Not enough RAM available to buffer future message - ignore" ) );
                     goto exit;
                 }
-                else if( ret != 0 )
+                hs_buf->data_len = reassembly_buf_sz;
-                    return( ret );
 
                 /* Prepare final header: copy msg_type, length and message_seq,
                  * then add standardised fragment_offset and fragment_length */
@@ -4526,6 +4560,8 @@ static int ssl_buffer_message( mbedtls_ssl_context *ssl )
                 memcpy( hs_buf->data + 9, hs_buf->data + 1, 3 );
 
                 hs_buf->is_valid = 1;
+
+                hs->buffering.total_bytes_buffered += reassembly_buf_sz;
             }
             else
             {