nbd: implement TLS support in the protocol negotiation

This extends the NBD protocol handling code so that it is capable
of negotiating TLS support during the connection setup. This involves
requesting the STARTTLS protocol option before any other NBD options.

Signed-off-by: Daniel P. Berrange <berrange@redhat.com>
Message-Id: <1455129674-17255-14-git-send-email-berrange@redhat.com>
Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>

block/nbd-client.c

@@ -405,7 +405,10 @@ int nbd_client_init(BlockDriverState *bs, QIOChannelSocket *sioc,
     qio_channel_set_blocking(QIO_CHANNEL(sioc), true, NULL);
 
     ret = nbd_receive_negotiate(QIO_CHANNEL(sioc), export,
-                                &client->nbdflags, &client->size, errp);
+                                &client->nbdflags,
+                                NULL, NULL,
+                                &client->ioc,
+                                &client->size, errp);
     if (ret < 0) {
         logout("Failed to negotiate with the NBD server\n");
         return ret;
@@ -415,8 +418,11 @@ int nbd_client_init(BlockDriverState *bs, QIOChannelSocket *sioc,
     qemu_co_mutex_init(&client->free_sema);
     client->sioc = sioc;
     object_ref(OBJECT(client->sioc));
-    client->ioc = QIO_CHANNEL(sioc);
-    object_ref(OBJECT(client->ioc));
+
+    if (!client->ioc) {
+        client->ioc = QIO_CHANNEL(sioc);
+        object_ref(OBJECT(client->ioc));
+    }
 
     /* Now that we're connected, set the socket to be non-blocking and
      * kick the reply mechanism.  */

blockdev-nbd.c

@@ -34,7 +34,7 @@ static gboolean nbd_accept(QIOChannel *ioc, GIOCondition condition,
         return TRUE;
     }
 
-    nbd_client_new(NULL, cioc, nbd_client_put);
+    nbd_client_new(NULL, cioc, NULL, NULL, nbd_client_put);
     object_unref(OBJECT(cioc));
     return TRUE;
 }

include/block/nbd.h

@@ -24,6 +24,7 @@
 #include "qemu-common.h"
 #include "qemu/option.h"
 #include "io/channel-socket.h"
+#include "crypto/tlscreds.h"
 
 struct nbd_request {
     uint32_t magic;
@@ -56,7 +57,10 @@ struct nbd_reply {
 #define NBD_REP_ACK             (1)             /* Data sending finished. */
 #define NBD_REP_SERVER          (2)             /* Export description. */
 #define NBD_REP_ERR_UNSUP       ((UINT32_C(1) << 31) | 1) /* Unknown option. */
+#define NBD_REP_ERR_POLICY      ((UINT32_C(1) << 31) | 2) /* Server denied */
 #define NBD_REP_ERR_INVALID     ((UINT32_C(1) << 31) | 3) /* Invalid length. */
+#define NBD_REP_ERR_TLS_REQD    ((UINT32_C(1) << 31) | 5) /* TLS required */
+
 
 #define NBD_CMD_MASK_COMMAND	0x0000ffff
 #define NBD_CMD_FLAG_FUA	(1 << 16)
@@ -81,6 +85,8 @@ ssize_t nbd_wr_syncv(QIOChannel *ioc,
                      size_t length,
                      bool do_read);
 int nbd_receive_negotiate(QIOChannel *ioc, const char *name, uint32_t *flags,
+                          QCryptoTLSCreds *tlscreds, const char *hostname,
+                          QIOChannel **outioc,
                           off_t *size, Error **errp);
 int nbd_init(int fd, QIOChannelSocket *sioc, uint32_t flags, off_t size);
 ssize_t nbd_send_request(QIOChannel *ioc, struct nbd_request *request);
@@ -106,6 +112,8 @@ void nbd_export_close_all(void);
 
 void nbd_client_new(NBDExport *exp,
                     QIOChannelSocket *sioc,
+                    QCryptoTLSCreds *tlscreds,
+                    const char *tlsaclname,
                     void (*close)(NBDClient *));
 void nbd_client_get(NBDClient *client);
 void nbd_client_put(NBDClient *client);

nbd/client.c

@@ -83,10 +83,18 @@ static int nbd_handle_reply_err(uint32_t opt, uint32_t type, Error **errp)
         error_setg(errp, "Unsupported option type %x", opt);
         break;
 
+    case NBD_REP_ERR_POLICY:
+        error_setg(errp, "Denied by server for option %x", opt);
+        break;
+
     case NBD_REP_ERR_INVALID:
         error_setg(errp, "Invalid data length for option %x", opt);
         break;
 
+    case NBD_REP_ERR_TLS_REQD:
+        error_setg(errp, "TLS negotiation required before option %x", opt);
+        break;
+
     default:
         error_setg(errp, "Unknown error code when asking for option %x", opt);
         break;
@@ -242,17 +250,127 @@ static int nbd_receive_query_exports(QIOChannel *ioc,
     return 0;
 }
 
+static QIOChannel *nbd_receive_starttls(QIOChannel *ioc,
+                                        QCryptoTLSCreds *tlscreds,
+                                        const char *hostname, Error **errp)
+{
+    uint64_t magic = cpu_to_be64(NBD_OPTS_MAGIC);
+    uint32_t opt = cpu_to_be32(NBD_OPT_STARTTLS);
+    uint32_t length = 0;
+    uint32_t type;
+    QIOChannelTLS *tioc;
+    struct NBDTLSHandshakeData data = { 0 };
+
+    TRACE("Requesting TLS from server");
+    if (write_sync(ioc, &magic, sizeof(magic)) != sizeof(magic)) {
+        error_setg(errp, "Failed to send option magic");
+        return NULL;
+    }
+
+    if (write_sync(ioc, &opt, sizeof(opt)) != sizeof(opt)) {
+        error_setg(errp, "Failed to send option number");
+        return NULL;
+    }
+
+    if (write_sync(ioc, &length, sizeof(length)) != sizeof(length)) {
+        error_setg(errp, "Failed to send option length");
+        return NULL;
+    }
+
+    TRACE("Getting TLS reply from server1");
+    if (read_sync(ioc, &magic, sizeof(magic)) != sizeof(magic)) {
+        error_setg(errp, "failed to read option magic");
+        return NULL;
+    }
+    magic = be64_to_cpu(magic);
+    if (magic != NBD_REP_MAGIC) {
+        error_setg(errp, "Unexpected option magic");
+        return NULL;
+    }
+    TRACE("Getting TLS reply from server2");
+    if (read_sync(ioc, &opt, sizeof(opt)) != sizeof(opt)) {
+        error_setg(errp, "failed to read option");
+        return NULL;
+    }
+    opt = be32_to_cpu(opt);
+    if (opt != NBD_OPT_STARTTLS) {
+        error_setg(errp, "Unexpected option type %x expected %x",
+                   opt, NBD_OPT_STARTTLS);
+        return NULL;
+    }
+
+    TRACE("Getting TLS reply from server");
+    if (read_sync(ioc, &type, sizeof(type)) != sizeof(type)) {
+        error_setg(errp, "failed to read option type");
+        return NULL;
+    }
+    type = be32_to_cpu(type);
+    if (type != NBD_REP_ACK) {
+        error_setg(errp, "Server rejected request to start TLS %x",
+                   type);
+        return NULL;
+    }
+
+    TRACE("Getting TLS reply from server");
+    if (read_sync(ioc, &length, sizeof(length)) != sizeof(length)) {
+        error_setg(errp, "failed to read option length");
+        return NULL;
+    }
+    length = be32_to_cpu(length);
+    if (length != 0) {
+        error_setg(errp, "Start TLS reponse was not zero %x",
+                   length);
+        return NULL;
+    }
+
+    TRACE("TLS request approved, setting up TLS");
+    tioc = qio_channel_tls_new_client(ioc, tlscreds, hostname, errp);
+    if (!tioc) {
+        return NULL;
+    }
+    data.loop = g_main_loop_new(g_main_context_default(), FALSE);
+    TRACE("Starting TLS hanshake");
+    qio_channel_tls_handshake(tioc,
+                              nbd_tls_handshake,
+                              &data,
+                              NULL);
+
+    if (!data.complete) {
+        g_main_loop_run(data.loop);
+    }
+    g_main_loop_unref(data.loop);
+    if (data.error) {
+        error_propagate(errp, data.error);
+        object_unref(OBJECT(tioc));
+        return NULL;
+    }
+
+    return QIO_CHANNEL(tioc);
+}
+
+
 int nbd_receive_negotiate(QIOChannel *ioc, const char *name, uint32_t *flags,
+                          QCryptoTLSCreds *tlscreds, const char *hostname,
+                          QIOChannel **outioc,
                           off_t *size, Error **errp)
 {
     char buf[256];
     uint64_t magic, s;
     int rc;
 
-    TRACE("Receiving negotiation.");
+    TRACE("Receiving negotiation tlscreds=%p hostname=%s.",
+          tlscreds, hostname ? hostname : "<null>");
 
     rc = -EINVAL;
 
+    if (outioc) {
+        *outioc = NULL;
+    }
+    if (tlscreds && !outioc) {
+        error_setg(errp, "Output I/O channel required for TLS");
+        goto fail;
+    }
+
     if (read_sync(ioc, buf, 8) != 8) {
         error_setg(errp, "Failed to read data");
         goto fail;
@@ -314,6 +432,18 @@ int nbd_receive_negotiate(QIOChannel *ioc, const char *name, uint32_t *flags,
             error_setg(errp, "Failed to send clientflags field");
             goto fail;
         }
+        if (tlscreds) {
+            if (fixedNewStyle) {
+                *outioc = nbd_receive_starttls(ioc, tlscreds, hostname, errp);
+                if (!*outioc) {
+                    goto fail;
+                }
+                ioc = *outioc;
+            } else {
+                error_setg(errp, "Server does not support STARTTLS");
+                goto fail;
+            }
+        }
         if (!name) {
             TRACE("Using default NBD export name \"\"");
             name = "";
@@ -371,6 +501,10 @@ int nbd_receive_negotiate(QIOChannel *ioc, const char *name, uint32_t *flags,
             error_setg(errp, "Server does not support export names");
             goto fail;
         }
+        if (tlscreds) {
+            error_setg(errp, "Server does not support STARTTLS");
+            goto fail;
+        }
 
         if (read_sync(ioc, &s, sizeof(s)) != sizeof(s)) {
             error_setg(errp, "Failed to read export length");

nbd/common.c

@@ -75,3 +75,18 @@ ssize_t nbd_wr_syncv(QIOChannel *ioc,
     g_free(local_iov_head);
     return done;
 }
+
+
+void nbd_tls_handshake(Object *src,
+                       Error *err,
+                       void *opaque)
+{
+    struct NBDTLSHandshakeData *data = opaque;
+
+    if (err) {
+        TRACE("TLS failed %s", error_get_pretty(err));
+        data->error = error_copy(err);
+    }
+    data->complete = true;
+    g_main_loop_quit(data->loop);
+}

nbd/nbd-internal.h

@@ -11,6 +11,7 @@
 #define NBD_INTERNAL_H
 #include "block/nbd.h"
 #include "sysemu/block-backend.h"
+#include "io/channel-tls.h"
 
 #include "qemu/coroutine.h"
 #include "qemu/iov.h"
@@ -78,6 +79,8 @@
 #define NBD_OPT_EXPORT_NAME     (1)
 #define NBD_OPT_ABORT           (2)
 #define NBD_OPT_LIST            (3)
+#define NBD_OPT_PEEK_EXPORT     (4)
+#define NBD_OPT_STARTTLS        (5)
 
 /* NBD errors are based on errno numbers, so there is a 1:1 mapping,
  * but only a limited set of errno values is specified in the protocol.
@@ -108,4 +111,15 @@ static inline ssize_t write_sync(QIOChannel *ioc, void *buffer, size_t size)
     return nbd_wr_syncv(ioc, &iov, 1, 0, size, false);
 }
 
+struct NBDTLSHandshakeData {
+    GMainLoop *loop;
+    bool complete;
+    Error *error;
+};
+
+
+void nbd_tls_handshake(Object *src,
+                       Error *err,
+                       void *opaque);
+
 #endif

nbd/server.c

@@ -76,6 +76,8 @@ struct NBDClient {
     void (*close)(NBDClient *client);
 
     NBDExport *exp;
+    QCryptoTLSCreds *tlscreds;
+    char *tlsaclname;
     QIOChannelSocket *sioc; /* The underlying data channel */
     QIOChannel *ioc; /* The current I/O channel which may differ (eg TLS) */
 
@@ -192,6 +194,8 @@ static int nbd_negotiate_send_rep(QIOChannel *ioc, uint32_t type, uint32_t opt)
     uint64_t magic;
     uint32_t len;
 
+    TRACE("Reply opt=%x type=%x", type, opt);
+
     magic = cpu_to_be64(NBD_REP_MAGIC);
     if (nbd_negotiate_write(ioc, &magic, sizeof(magic)) != sizeof(magic)) {
         LOG("write failed (rep magic)");
@@ -310,6 +314,55 @@ fail:
     return rc;
 }
 
+
+static QIOChannel *nbd_negotiate_handle_starttls(NBDClient *client,
+                                                 uint32_t length)
+{
+    QIOChannel *ioc;
+    QIOChannelTLS *tioc;
+    struct NBDTLSHandshakeData data = { 0 };
+
+    TRACE("Setting up TLS");
+    ioc = client->ioc;
+    if (length) {
+        if (nbd_negotiate_drop_sync(ioc, length) != length) {
+            return NULL;
+        }
+        nbd_negotiate_send_rep(ioc, NBD_REP_ERR_INVALID, NBD_OPT_STARTTLS);
+        return NULL;
+    }
+
+    nbd_negotiate_send_rep(client->ioc, NBD_REP_ACK, NBD_OPT_STARTTLS);
+
+    tioc = qio_channel_tls_new_server(ioc,
+                                      client->tlscreds,
+                                      client->tlsaclname,
+                                      NULL);
+    if (!tioc) {
+        return NULL;
+    }
+
+    TRACE("Starting TLS handshake");
+    data.loop = g_main_loop_new(g_main_context_default(), FALSE);
+    qio_channel_tls_handshake(tioc,
+                              nbd_tls_handshake,
+                              &data,
+                              NULL);
+
+    if (!data.complete) {
+        g_main_loop_run(data.loop);
+    }
+    g_main_loop_unref(data.loop);
+    if (data.error) {
+        object_unref(OBJECT(tioc));
+        error_free(data.error);
+        return NULL;
+    }
+
+    return QIO_CHANNEL(tioc);
+}
+
+
 static int nbd_negotiate_options(NBDClient *client)
 {
     uint32_t flags;
@@ -377,7 +430,30 @@ static int nbd_negotiate_options(NBDClient *client)
         length = be32_to_cpu(length);
 
         TRACE("Checking option 0x%x", clientflags);
-        if (fixedNewstyle) {
+        if (client->tlscreds &&
+            client->ioc == (QIOChannel *)client->sioc) {
+            QIOChannel *tioc;
+            if (!fixedNewstyle) {
+                TRACE("Unsupported option 0x%x", clientflags);
+                return -EINVAL;
+            }
+            switch (clientflags) {
+            case NBD_OPT_STARTTLS:
+                tioc = nbd_negotiate_handle_starttls(client, length);
+                if (!tioc) {
+                    return -EIO;
+                }
+                object_unref(OBJECT(client->ioc));
+                client->ioc = QIO_CHANNEL(tioc);
+                break;
+
+            default:
+                TRACE("Option 0x%x not permitted before TLS", clientflags);
+                nbd_negotiate_send_rep(client->ioc, NBD_REP_ERR_TLS_REQD,
+                                       clientflags);
+                return -EINVAL;
+            }
+        } else if (fixedNewstyle) {
             switch (clientflags) {
             case NBD_OPT_LIST:
                 ret = nbd_negotiate_handle_list(client, length);
@@ -392,6 +468,17 @@ static int nbd_negotiate_options(NBDClient *client)
             case NBD_OPT_EXPORT_NAME:
                 return nbd_negotiate_handle_export_name(client, length);
 
+            case NBD_OPT_STARTTLS:
+                if (client->tlscreds) {
+                    TRACE("TLS already enabled");
+                    nbd_negotiate_send_rep(client->ioc, NBD_REP_ERR_INVALID,
+                                           clientflags);
+                } else {
+                    TRACE("TLS not configured");
+                    nbd_negotiate_send_rep(client->ioc, NBD_REP_ERR_POLICY,
+                                           clientflags);
+                }
+                return -EINVAL;
             default:
                 TRACE("Unsupported option 0x%x", clientflags);
                 nbd_negotiate_send_rep(client->ioc, NBD_REP_ERR_UNSUP,
@@ -427,8 +514,9 @@ static coroutine_fn int nbd_negotiate(NBDClientNewData *data)
     int rc;
     const int myflags = (NBD_FLAG_HAS_FLAGS | NBD_FLAG_SEND_TRIM |
                          NBD_FLAG_SEND_FLUSH | NBD_FLAG_SEND_FUA);
+    bool oldStyle;
 
-    /* Negotiation header without options:
+    /* Old style negotiation header without options
         [ 0 ..   7]   passwd       ("NBDMAGIC")
         [ 8 ..  15]   magic        (NBD_CLIENT_MAGIC)
         [16 ..  23]   size
@@ -436,12 +524,11 @@ static coroutine_fn int nbd_negotiate(NBDClientNewData *data)
         [26 ..  27]   export flags
         [28 .. 151]   reserved     (0)
 
-       Negotiation header with options, part 1:
+       New style negotiation header with options
         [ 0 ..   7]   passwd       ("NBDMAGIC")
         [ 8 ..  15]   magic        (NBD_OPTS_MAGIC)
         [16 ..  17]   server flags (0)
-
-       part 2 (after options are sent):
+        ....options sent....
         [18 ..  25]   size
         [26 ..  27]   export flags
         [28 .. 151]   reserved     (0)
@@ -453,7 +540,9 @@ static coroutine_fn int nbd_negotiate(NBDClientNewData *data)
     TRACE("Beginning negotiation.");
     memset(buf, 0, sizeof(buf));
     memcpy(buf, "NBDMAGIC", 8);
-    if (client->exp) {
+
+    oldStyle = client->exp != NULL && !client->tlscreds;
+    if (oldStyle) {
         assert ((client->exp->nbdflags & ~65535) == 0);
         stq_be_p(buf + 8, NBD_CLIENT_MAGIC);
         stq_be_p(buf + 16, client->exp->size);
@@ -463,7 +552,11 @@ static coroutine_fn int nbd_negotiate(NBDClientNewData *data)
         stw_be_p(buf + 16, NBD_FLAG_FIXED_NEWSTYLE);
     }
 
-    if (client->exp) {
+    if (oldStyle) {
+        if (client->tlscreds) {
+            TRACE("TLS cannot be enabled with oldstyle protocol");
+            goto fail;
+        }
         if (nbd_negotiate_write(client->ioc, buf, sizeof(buf)) != sizeof(buf)) {
             LOG("write failed");
             goto fail;
@@ -602,6 +695,10 @@ void nbd_client_put(NBDClient *client)
         nbd_unset_handlers(client);
         object_unref(OBJECT(client->sioc));
         object_unref(OBJECT(client->ioc));
+        if (client->tlscreds) {
+            object_unref(OBJECT(client->tlscreds));
+        }
+        g_free(client->tlsaclname);
         if (client->exp) {
             QTAILQ_REMOVE(&client->exp->clients, client, next);
             nbd_export_put(client->exp);
@@ -1150,6 +1247,8 @@ out:
 
 void nbd_client_new(NBDExport *exp,
                     QIOChannelSocket *sioc,
+                    QCryptoTLSCreds *tlscreds,
+                    const char *tlsaclname,
                     void (*close_fn)(NBDClient *))
 {
     NBDClient *client;
@@ -1158,6 +1257,11 @@ void nbd_client_new(NBDExport *exp,
     client = g_malloc0(sizeof(NBDClient));
     client->refcount = 1;
     client->exp = exp;
+    client->tlscreds = tlscreds;
+    if (tlscreds) {
+        object_ref(OBJECT(client->tlscreds));
+    }
+    client->tlsaclname = g_strdup(tlsaclname);
     client->sioc = sioc;
     object_ref(OBJECT(client->sioc));
     client->ioc = QIO_CHANNEL(sioc);

qemu-nbd.c

@@ -251,6 +251,7 @@ static void *nbd_client_thread(void *arg)
     }
 
     ret = nbd_receive_negotiate(QIO_CHANNEL(sioc), NULL, &nbdflags,
+                                NULL, NULL, NULL,
                                 &size, &local_error);
     if (ret < 0) {
         if (local_error) {
@@ -340,7 +341,8 @@ static gboolean nbd_accept(QIOChannel *ioc, GIOCondition cond, gpointer opaque)
 
     nb_fds++;
     nbd_update_server_watch();
-    nbd_client_new(newproto ? NULL : exp, cioc, nbd_client_closed);
+    nbd_client_new(newproto ? NULL : exp, cioc,
+                   NULL, NULL, nbd_client_closed);
     object_unref(OBJECT(cioc));
 
     return TRUE;