This series is mostly about 9p request cancellation. It fixes a

long standing bug (read "specification violation") where the server
 would send an invalid response when the client has cancelled an
 in-flight request. This was causing annoying spurious EINTR returns
 in linux. The fix comes with some related testing in QTEST.
 
 Other patches are code cleanup and improvements.
 -----BEGIN PGP SIGNATURE-----
 
 iQIzBAABCAAdFiEEtIKLr5QxQM7yo0kQcdTV5YIvc9YFAlp0OmMACgkQcdTV5YIv
 c9bM2w/+ORsH7ifYVoPZDwPdhLAVg8t4aU7ak7kWSOeuTZhE8WA6GL8T4rjnhZZ4
 PbFZTye0fOsZZaT30sHUIwI/CGewvdx0kMJuX339XfwaWvsrF/8tHFA8QdreclVf
 YlLJwD2yZRujWPCU8k5D3hUfH2YD/w68b9Hv7K2hgsPw1Zoq7pLi/B2/BK/62G1d
 35nFG97xDmcScSybrIeLwaCIKW/wgjHsWgUgAVMpgaSwtu/JTtEG5JYCZLGO6HVa
 p67zdVQbCKQcdPLLllcrnGdWlA4CGxCBe+WMQyL3eJ2VDEH6XTBX8DiLstIVlXfj
 vXb7vjlsYxE6rdeiMXRZWDGK+jZIePmDsI6ILlpyL0rEphuBnjEMljCxyBnOp7gS
 dMTFmhfP4mApaYxeENlpQ5noDeQ/Jr1c9YeeBh9UnZ7r16/m3f2/sEIh+6QTL7YD
 iR7ZG5LF0eP5SJKoZNV0i4123A5I7v1ekn5N5q6Kf4PMcY11507lxZksPhckdPgp
 FPhsJJ1JQzWMcfE1mKeiZXq5Cys4EUhUK6UrOFp6tO38H/3YMuYMAz25D46Xk3Zx
 qGuptCaov9uPi+5uqKuG3aJ9nMIMW9ZJdMSWo2+kb4EX4hMiOewjOJtaKA11QeS+
 YoEr3szKtVnPUKsiqm+lhHL6ec5B8NQB7f8Hl88pkoLBGQzMaWU=
 =9T0i
 -----END PGP SIGNATURE-----

Merge remote-tracking branch 'remotes/gkurz/tags/for-upstream' into staging

This series is mostly about 9p request cancellation. It fixes a
long standing bug (read "specification violation") where the server
would send an invalid response when the client has cancelled an
in-flight request. This was causing annoying spurious EINTR returns
in linux. The fix comes with some related testing in QTEST.

Other patches are code cleanup and improvements.

# gpg: Signature made Fri 02 Feb 2018 10:16:03 GMT
# gpg:                using RSA key 71D4D5E5822F73D6
# gpg: Good signature from "Greg Kurz <groug@kaod.org>"
# gpg:                 aka "Gregory Kurz <gregory.kurz@free.fr>"
# gpg:                 aka "[jpeg image of size 3330]"
# Primary key fingerprint: B482 8BAF 9431 40CE F2A3  4910 71D4 D5E5 822F 73D6

* remotes/gkurz/tags/for-upstream:
  tests/virtio-9p: explicitly handle potential integer overflows
  tests: virtio-9p: add FLUSH operation test
  libqos/virtio: return length written into used descriptor
  tests: virtio-9p: add WRITE operation test
  tests: virtio-9p: add LOPEN operation test
  tests: virtio-9p: use the synth backend
  tests: virtio-9p: wait for completion in the test code
  tests: virtio-9p: move request tag to the test functions
  9pfs: Correctly handle cancelled requests
  9pfs: drop v9fs_register_transport()

Signed-off-by: Peter Maydell <peter.maydell@linaro.org>

hw/9pfs/9p-synth.c

@@ -19,6 +19,7 @@
 #include "qemu/rcu.h"
 #include "qemu/rcu_queue.h"
 #include "qemu/cutils.h"
+#include "sysemu/qtest.h"
 
 /* Root node for synth file system */
 static V9fsSynthNode synth_root = {
@@ -514,6 +515,26 @@ static int synth_unlinkat(FsContext *ctx, V9fsPath *dir,
     return -1;
 }
 
+static ssize_t v9fs_synth_qtest_write(void *buf, int len, off_t offset,
+                                      void *arg)
+{
+    return 1;
+}
+
+static ssize_t v9fs_synth_qtest_flush_write(void *buf, int len, off_t offset,
+                                            void *arg)
+{
+    bool should_block = !!*(uint8_t *)buf;
+
+    if (should_block) {
+        /* This will cause the server to call us again until we're cancelled */
+        errno = EINTR;
+        return -1;
+    }
+
+    return 1;
+}
+
 static int synth_init(FsContext *ctx, Error **errp)
 {
     QLIST_INIT(&synth_root.child);
@@ -527,6 +548,37 @@ static int synth_init(FsContext *ctx, Error **errp)
 
     /* Mark the subsystem is ready for use */
     synth_fs = 1;
+
+    if (qtest_enabled()) {
+        V9fsSynthNode *node = NULL;
+        int i, ret;
+
+        /* Directory hierarchy for WALK test */
+        for (i = 0; i < P9_MAXWELEM; i++) {
+            char *name = g_strdup_printf(QTEST_V9FS_SYNTH_WALK_FILE, i);
+
+            ret = qemu_v9fs_synth_mkdir(node, 0700, name, &node);
+            assert(!ret);
+            g_free(name);
+        }
+
+        /* File for LOPEN test */
+        ret = qemu_v9fs_synth_add_file(NULL, 0, QTEST_V9FS_SYNTH_LOPEN_FILE,
+                                       NULL, NULL, ctx);
+        assert(!ret);
+
+        /* File for WRITE test */
+        ret = qemu_v9fs_synth_add_file(NULL, 0, QTEST_V9FS_SYNTH_WRITE_FILE,
+                                       NULL, v9fs_synth_qtest_write, ctx);
+        assert(!ret);
+
+        /* File for FLUSH test */
+        ret = qemu_v9fs_synth_add_file(NULL, 0, QTEST_V9FS_SYNTH_FLUSH_FILE,
+                                       NULL, v9fs_synth_qtest_flush_write,
+                                       ctx);
+        assert(!ret);
+    }
+
     return 0;
 }
 

hw/9pfs/9p-synth.h

@@ -49,4 +49,17 @@ int qemu_v9fs_synth_add_file(V9fsSynthNode *parent, int mode,
                              const char *name, v9fs_synth_read read,
                              v9fs_synth_write write, void *arg);
 
+/* qtest stuff */
+
+#define QTEST_V9FS_SYNTH_WALK_FILE "WALK%d"
+#define QTEST_V9FS_SYNTH_LOPEN_FILE "LOPEN"
+#define QTEST_V9FS_SYNTH_WRITE_FILE "WRITE"
+
+/* Any write to the "FLUSH" file is handled one byte at a time by the
+ * backend. If the byte is zero, the backend returns success (ie, 1),
+ * otherwise it forces the server to try again forever. Thus allowing
+ * the client to cancel the request.
+ */
+#define QTEST_V9FS_SYNTH_FLUSH_FILE "FLUSH"
+
 #endif

hw/9pfs/9p.c

@@ -24,6 +24,7 @@
 #include "coth.h"
 #include "trace.h"
 #include "migration/blocker.h"
+#include "sysemu/qtest.h"
 
 int open_fd_hw;
 int total_open_fd;
@@ -630,6 +631,24 @@ static void coroutine_fn pdu_complete(V9fsPDU *pdu, ssize_t len)
     V9fsState *s = pdu->s;
     int ret;
 
+    /*
+     * The 9p spec requires that successfully cancelled pdus receive no reply.
+     * Sending a reply would confuse clients because they would
+     * assume that any EINTR is the actual result of the operation,
+     * rather than a consequence of the cancellation. However, if
+     * the operation completed (succesfully or with an error other
+     * than caused be cancellation), we do send out that reply, both
+     * for efficiency and to avoid confusing the rest of the state machine
+     * that assumes passing a non-error here will mean a successful
+     * transmission of the reply.
+     */
+    bool discard = pdu->cancelled && len == -EINTR;
+    if (discard) {
+        trace_v9fs_rcancel(pdu->tag, pdu->id);
+        pdu->size = 0;
+        goto out_notify;
+    }
+
     if (len < 0) {
         int err = -len;
         len = 7;
@@ -3485,7 +3504,8 @@ void pdu_submit(V9fsPDU *pdu, P9MsgHeader *hdr)
 }
 
 /* Returns 0 on success, 1 on failure. */
-int v9fs_device_realize_common(V9fsState *s, Error **errp)
+int v9fs_device_realize_common(V9fsState *s, const V9fsTransport *t,
+                               Error **errp)
 {
     int i, len;
     struct stat stat;
@@ -3493,6 +3513,9 @@ int v9fs_device_realize_common(V9fsState *s, Error **errp)
     V9fsPath path;
     int rc = 1;
 
+    assert(!s->transport);
+    s->transport = t;
+
     /* initialize pdu allocator */
     QLIST_INIT(&s->free_list);
     QLIST_INIT(&s->active_list);

hw/9pfs/9p.h

@@ -346,7 +346,8 @@ void v9fs_path_sprintf(V9fsPath *path, const char *fmt, ...);
 void v9fs_path_copy(V9fsPath *lhs, V9fsPath *rhs);
 int v9fs_name_to_path(V9fsState *s, V9fsPath *dirpath,
                       const char *name, V9fsPath *path);
-int v9fs_device_realize_common(V9fsState *s, Error **errp);
+int v9fs_device_realize_common(V9fsState *s, const V9fsTransport *t,
+                               Error **errp);
 void v9fs_device_unrealize_common(V9fsState *s, Error **errp);
 
 V9fsPDU *pdu_alloc(V9fsState *s);
@@ -366,11 +367,4 @@ struct V9fsTransport {
     void        (*push_and_notify)(V9fsPDU *pdu);
 };
 
-static inline int v9fs_register_transport(V9fsState *s, const V9fsTransport *t)
-{
-    assert(!s->transport);
-    s->transport = t;
-    return 0;
-}
-
 #endif

hw/9pfs/trace-events

@@ -1,6 +1,7 @@
 # See docs/devel/tracing.txt for syntax documentation.
 
 # hw/9pfs/virtio-9p.c
+v9fs_rcancel(uint16_t tag, uint8_t id) "tag %d id %d"
 v9fs_rerror(uint16_t tag, uint8_t id, int err) "tag %d id %d err %d"
 v9fs_version(uint16_t tag, uint8_t id, int32_t msize, char* version) "tag %d id %d msize %d version %s"
 v9fs_version_return(uint16_t tag, uint8_t id, int32_t msize, char* version) "tag %d id %d msize %d version %s"

hw/9pfs/virtio-9p-device.c

@@ -198,17 +198,13 @@ static void virtio_9p_device_realize(DeviceState *dev, Error **errp)
     V9fsVirtioState *v = VIRTIO_9P(dev);
     V9fsState *s = &v->state;
 
-    if (v9fs_device_realize_common(s, errp)) {
-        goto out;
+    if (v9fs_device_realize_common(s, &virtio_9p_transport, errp)) {
+        return;
     }
 
     v->config_size = sizeof(struct virtio_9p_config) + strlen(s->fsconf.tag);
     virtio_init(vdev, "virtio-9p", VIRTIO_ID_9P, v->config_size);
     v->vq = virtio_add_queue(vdev, MAX_REQ, handle_9p_output);
-    v9fs_register_transport(s, &virtio_9p_transport);
-
-out:
-    return;
 }
 
 static void virtio_9p_device_unrealize(DeviceState *dev, Error **errp)

hw/9pfs/xen-9p-backend.c

@@ -446,7 +446,6 @@ static int xen_9pfs_connect(struct XenDevice *xendev)
     xen_9pdev->id = s->fsconf.fsdev_id =
         g_strdup_printf("xen9p%d", xendev->dev);
     xen_9pdev->tag = s->fsconf.tag = xenstore_read_fe_str(xendev, "tag");
-    v9fs_register_transport(s, &xen_9p_transport);
     fsdev = qemu_opts_create(qemu_find_opts("fsdev"),
             s->fsconf.tag,
             1, NULL);
@@ -455,7 +454,7 @@ static int xen_9pfs_connect(struct XenDevice *xendev)
     qemu_opt_set(fsdev, "security_model", xen_9pdev->security_model, NULL);
     qemu_opts_set_id(fsdev, s->fsconf.fsdev_id);
     qemu_fsdev_add(fsdev);
-    v9fs_device_realize_common(s, NULL);
+    v9fs_device_realize_common(s, &xen_9p_transport, NULL);
 
     return 0;
 

tests/libqos/virtio.c

@@ -119,6 +119,8 @@ uint8_t qvirtio_wait_status_byte_no_isr(QVirtioDevice *d,
 /*
  * qvirtio_wait_used_elem:
  * @desc_idx: The next expected vq->desc[] index in the used ring
+ * @len: A pointer that is filled with the length written into the buffer, may
+ *       be NULL
  * @timeout_us: How many microseconds to wait before failing
  *
  * This function waits for the next completed request on the used ring.
@@ -126,6 +128,7 @@ uint8_t qvirtio_wait_status_byte_no_isr(QVirtioDevice *d,
 void qvirtio_wait_used_elem(QVirtioDevice *d,
                             QVirtQueue *vq,
                             uint32_t desc_idx,
+                            uint32_t *len,
                             gint64 timeout_us)
 {
     gint64 start_time = g_get_monotonic_time();
@@ -136,7 +139,7 @@ void qvirtio_wait_used_elem(QVirtioDevice *d,
         clock_step(100);
 
         if (d->bus->get_queue_isr_status(d, vq) &&
-            qvirtqueue_get_buf(vq, &got_desc_idx)) {
+            qvirtqueue_get_buf(vq, &got_desc_idx, len)) {
             g_assert_cmpint(got_desc_idx, ==, desc_idx);
             return;
         }
@@ -304,30 +307,36 @@ void qvirtqueue_kick(QVirtioDevice *d, QVirtQueue *vq, uint32_t free_head)
 /*
  * qvirtqueue_get_buf:
  * @desc_idx: A pointer that is filled with the vq->desc[] index, may be NULL
+ * @len: A pointer that is filled with the length written into the buffer, may
+ *       be NULL
  *
  * This function gets the next used element if there is one ready.
  *
  * Returns: true if an element was ready, false otherwise
  */
-bool qvirtqueue_get_buf(QVirtQueue *vq, uint32_t *desc_idx)
+bool qvirtqueue_get_buf(QVirtQueue *vq, uint32_t *desc_idx, uint32_t *len)
 {
     uint16_t idx;
+    uint64_t elem_addr;
 
     idx = readw(vq->used + offsetof(struct vring_used, idx));
     if (idx == vq->last_used_idx) {
         return false;
     }
 
-    if (desc_idx) {
-        uint64_t elem_addr;
+    elem_addr = vq->used +
+        offsetof(struct vring_used, ring) +
+        (vq->last_used_idx % vq->size) *
+        sizeof(struct vring_used_elem);
 
-        elem_addr = vq->used +
-                    offsetof(struct vring_used, ring) +
-                    (vq->last_used_idx % vq->size) *
-                    sizeof(struct vring_used_elem);
+    if (desc_idx) {
         *desc_idx = readl(elem_addr + offsetof(struct vring_used_elem, id));
     }
 
+    if (len) {
+        *len = readw(elem_addr + offsetof(struct vring_used_elem, len));
+    }
+
     vq->last_used_idx++;
     return true;
 }

tests/libqos/virtio.h

@@ -124,6 +124,7 @@ uint8_t qvirtio_wait_status_byte_no_isr(QVirtioDevice *d,
 void qvirtio_wait_used_elem(QVirtioDevice *d,
                             QVirtQueue *vq,
                             uint32_t desc_idx,
+                            uint32_t *len,
                             gint64 timeout_us);
 void qvirtio_wait_config_isr(QVirtioDevice *d, gint64 timeout_us);
 QVirtQueue *qvirtqueue_setup(QVirtioDevice *d,
@@ -140,7 +141,7 @@ uint32_t qvirtqueue_add(QVirtQueue *vq, uint64_t data, uint32_t len, bool write,
                                                                     bool next);
 uint32_t qvirtqueue_add_indirect(QVirtQueue *vq, QVRingIndirectDesc *indirect);
 void qvirtqueue_kick(QVirtioDevice *d, QVirtQueue *vq, uint32_t free_head);
-bool qvirtqueue_get_buf(QVirtQueue *vq, uint32_t *desc_idx);
+bool qvirtqueue_get_buf(QVirtQueue *vq, uint32_t *desc_idx, uint32_t *len);
 
 void qvirtqueue_set_used_event(QVirtQueue *vq, uint16_t idx);
 

tests/virtio-9p-test.c

@@ -17,6 +17,7 @@
 #include "standard-headers/linux/virtio_ids.h"
 #include "standard-headers/linux/virtio_pci.h"
 #include "hw/9pfs/9p.h"
+#include "hw/9pfs/9p-synth.h"
 
 #define QVIRTIO_9P_TIMEOUT_US (10 * 1000 * 1000)
 
@@ -26,24 +27,19 @@ typedef struct {
     QVirtioDevice *dev;
     QOSState *qs;
     QVirtQueue *vq;
-    char *test_share;
-    uint16_t p9_req_tag;
 } QVirtIO9P;
 
 static QVirtIO9P *qvirtio_9p_start(const char *driver)
 {
     const char *arch = qtest_get_arch();
-    const char *cmd = "-fsdev local,id=fsdev0,security_model=none,path=%s "
+    const char *cmd = "-fsdev synth,id=fsdev0 "
                       "-device %s,fsdev=fsdev0,mount_tag=%s";
     QVirtIO9P *v9p = g_new0(QVirtIO9P, 1);
 
-    v9p->test_share = g_strdup("/tmp/qtest.XXXXXX");
-    g_assert_nonnull(mkdtemp(v9p->test_share));
-
     if (strcmp(arch, "i386") == 0 || strcmp(arch, "x86_64") == 0) {
-        v9p->qs = qtest_pc_boot(cmd, v9p->test_share, driver, mount_tag);
+        v9p->qs = qtest_pc_boot(cmd, driver, mount_tag);
     } else if (strcmp(arch, "ppc64") == 0) {
-        v9p->qs = qtest_spapr_boot(cmd, v9p->test_share, driver, mount_tag);
+        v9p->qs = qtest_spapr_boot(cmd, driver, mount_tag);
     } else {
         g_printerr("virtio-9p tests are only available on x86 or ppc64\n");
         exit(EXIT_FAILURE);
@@ -55,8 +51,6 @@ static QVirtIO9P *qvirtio_9p_start(const char *driver)
 static void qvirtio_9p_stop(QVirtIO9P *v9p)
 {
     qtest_shutdown(v9p->qs);
-    rmdir(v9p->test_share);
-    g_free(v9p->test_share);
     g_free(v9p);
 }
 
@@ -156,6 +150,13 @@ static void v9fs_uint32_write(P9Req *req, uint32_t val)
     v9fs_memwrite(req, &le_val, 4);
 }
 
+static void v9fs_uint64_write(P9Req *req, uint64_t val)
+{
+    uint64_t le_val = cpu_to_le64(val);
+
+    v9fs_memwrite(req, &le_val, 8);
+}
+
 static void v9fs_uint32_read(P9Req *req, uint32_t *val)
 {
     v9fs_memread(req, val, 4);
@@ -167,7 +168,7 @@ static uint16_t v9fs_string_size(const char *string)
 {
     size_t len = strlen(string);
 
-    g_assert_cmpint(len, <=, UINT16_MAX);
+    g_assert_cmpint(len, <=, UINT16_MAX - 2);
 
     return 2 + len;
 }
@@ -208,17 +209,20 @@ static P9Req *v9fs_req_init(QVirtIO9P *v9p, uint32_t size, uint8_t id,
                             uint16_t tag)
 {
     P9Req *req = g_new0(P9Req, 1);
-    uint32_t t_size = 7 + size; /* 9P header has well-known size of 7 bytes */
+    uint32_t total_size = 7; /* 9P header has well-known size of 7 bytes */
     P9Hdr hdr = {
-        .size = cpu_to_le32(t_size),
         .id = id,
         .tag = cpu_to_le16(tag)
     };
 
-    g_assert_cmpint(t_size, <=, P9_MAX_SIZE);
+    g_assert_cmpint(total_size, <=, UINT32_MAX - size);
+    total_size += size;
+    hdr.size = cpu_to_le32(total_size);
+
+    g_assert_cmpint(total_size, <=, P9_MAX_SIZE);
 
     req->v9p = v9p;
-    req->t_size = t_size;
+    req->t_size = total_size;
     req->t_msg = guest_alloc(v9p->qs->alloc, req->t_size);
     v9fs_memwrite(req, &hdr, 7);
     req->tag = tag;
@@ -244,16 +248,23 @@ static const char *rmessage_name(uint8_t id)
         id == P9_RVERSION ? "RVERSION" :
         id == P9_RATTACH ? "RATTACH" :
         id == P9_RWALK ? "RWALK" :
+        id == P9_RLOPEN ? "RLOPEN" :
+        id == P9_RWRITE ? "RWRITE" :
+        id == P9_RFLUSH ? "RFLUSH" :
         "<unknown>";
 }
 
-static void v9fs_req_recv(P9Req *req, uint8_t id)
+static void v9fs_req_wait_for_reply(P9Req *req, uint32_t *len)
 {
     QVirtIO9P *v9p = req->v9p;
-    P9Hdr hdr;
 
-    qvirtio_wait_used_elem(v9p->dev, v9p->vq, req->free_head,
+    qvirtio_wait_used_elem(v9p->dev, v9p->vq, req->free_head, len,
                            QVIRTIO_9P_TIMEOUT_US);
+}
+
+static void v9fs_req_recv(P9Req *req, uint8_t id)
+{
+    P9Hdr hdr;
 
     v9fs_memread(req, &hdr, 7);
     hdr.size = ldl_le_p(&hdr.size);
@@ -294,10 +305,16 @@ static void v9fs_rlerror(P9Req *req, uint32_t *err)
 }
 
 /* size[4] Tversion tag[2] msize[4] version[s] */
-static P9Req *v9fs_tversion(QVirtIO9P *v9p, uint32_t msize, const char *version)
+static P9Req *v9fs_tversion(QVirtIO9P *v9p, uint32_t msize, const char *version,
+                            uint16_t tag)
 {
-    P9Req *req = v9fs_req_init(v9p, 4 + v9fs_string_size(version), P9_TVERSION,
-                               P9_NOTAG);
+    P9Req *req;
+    uint32_t body_size = 4;
+    uint16_t string_size = v9fs_string_size(version);
+
+    g_assert_cmpint(body_size, <=, UINT32_MAX - string_size);
+    body_size += string_size;
+    req = v9fs_req_init(v9p, body_size, P9_TVERSION, tag);
 
     v9fs_uint32_write(req, msize);
     v9fs_string_write(req, version);
@@ -323,12 +340,12 @@ static void v9fs_rversion(P9Req *req, uint16_t *len, char **version)
 }
 
 /* size[4] Tattach tag[2] fid[4] afid[4] uname[s] aname[s] n_uname[4] */
-static P9Req *v9fs_tattach(QVirtIO9P *v9p, uint32_t fid, uint32_t n_uname)
+static P9Req *v9fs_tattach(QVirtIO9P *v9p, uint32_t fid, uint32_t n_uname,
+                           uint16_t tag)
 {
     const char *uname = ""; /* ignored by QEMU */
     const char *aname = ""; /* ignored by QEMU */
-    P9Req *req = v9fs_req_init(v9p, 4 + 4 + 2 + 2 + 4, P9_TATTACH,
-                               ++(v9p->p9_req_tag));
+    P9Req *req = v9fs_req_init(v9p, 4 + 4 + 2 + 2 + 4, P9_TATTACH, tag);
 
     v9fs_uint32_write(req, fid);
     v9fs_uint32_write(req, P9_NOFID);
@@ -353,16 +370,19 @@ static void v9fs_rattach(P9Req *req, v9fs_qid *qid)
 
 /* size[4] Twalk tag[2] fid[4] newfid[4] nwname[2] nwname*(wname[s]) */
 static P9Req *v9fs_twalk(QVirtIO9P *v9p, uint32_t fid, uint32_t newfid,
-                         uint16_t nwname, char *const wnames[])
+                         uint16_t nwname, char *const wnames[], uint16_t tag)
 {
     P9Req *req;
     int i;
-    uint32_t size = 4 + 4 + 2;
+    uint32_t body_size = 4 + 4 + 2;
 
     for (i = 0; i < nwname; i++) {
-        size += v9fs_string_size(wnames[i]);
+        uint16_t wname_size = v9fs_string_size(wnames[i]);
+
+        g_assert_cmpint(body_size, <=, UINT32_MAX - wname_size);
+        body_size += wname_size;
     }
-    req = v9fs_req_init(v9p,  size, P9_TWALK, ++(v9p->p9_req_tag));
+    req = v9fs_req_init(v9p,  body_size, P9_TWALK, tag);
     v9fs_uint32_write(req, fid);
     v9fs_uint32_write(req, newfid);
     v9fs_uint16_write(req, nwname);
@@ -390,6 +410,80 @@ static void v9fs_rwalk(P9Req *req, uint16_t *nwqid, v9fs_qid **wqid)
     v9fs_req_free(req);
 }
 
+/* size[4] Tlopen tag[2] fid[4] flags[4] */
+static P9Req *v9fs_tlopen(QVirtIO9P *v9p, uint32_t fid, uint32_t flags,
+                          uint16_t tag)
+{
+    P9Req *req;
+
+    req = v9fs_req_init(v9p,  4 + 4, P9_TLOPEN, tag);
+    v9fs_uint32_write(req, fid);
+    v9fs_uint32_write(req, flags);
+    v9fs_req_send(req);
+    return req;
+}
+
+/* size[4] Rlopen tag[2] qid[13] iounit[4] */
+static void v9fs_rlopen(P9Req *req, v9fs_qid *qid, uint32_t *iounit)
+{
+    v9fs_req_recv(req, P9_RLOPEN);
+    if (qid) {
+        v9fs_memread(req, qid, 13);
+    } else {
+        v9fs_memskip(req, 13);
+    }
+    if (iounit) {
+        v9fs_uint32_read(req, iounit);
+    }
+    v9fs_req_free(req);
+}
+
+/* size[4] Twrite tag[2] fid[4] offset[8] count[4] data[count] */
+static P9Req *v9fs_twrite(QVirtIO9P *v9p, uint32_t fid, uint64_t offset,
+                          uint32_t count, const void *data, uint16_t tag)
+{
+    P9Req *req;
+    uint32_t body_size = 4 + 8 + 4;
+
+    g_assert_cmpint(body_size, <=, UINT32_MAX - count);
+    body_size += count;
+    req = v9fs_req_init(v9p,  body_size, P9_TWRITE, tag);
+    v9fs_uint32_write(req, fid);
+    v9fs_uint64_write(req, offset);
+    v9fs_uint32_write(req, count);
+    v9fs_memwrite(req, data, count);
+    v9fs_req_send(req);
+    return req;
+}
+
+/* size[4] Rwrite tag[2] count[4] */
+static void v9fs_rwrite(P9Req *req, uint32_t *count)
+{
+    v9fs_req_recv(req, P9_RWRITE);
+    if (count) {
+        v9fs_uint32_read(req, count);
+    }
+    v9fs_req_free(req);
+}
+
+/* size[4] Tflush tag[2] oldtag[2] */
+static P9Req *v9fs_tflush(QVirtIO9P *v9p, uint16_t oldtag, uint16_t tag)
+{
+    P9Req *req;
+
+    req = v9fs_req_init(v9p,  2, P9_TFLUSH, tag);
+    v9fs_uint32_write(req, oldtag);
+    v9fs_req_send(req);
+    return req;
+}
+
+/* size[4] Rflush tag[2] */
+static void v9fs_rflush(P9Req *req)
+{
+    v9fs_req_recv(req, P9_RFLUSH);
+    v9fs_req_free(req);
+}
+
 static void fs_version(QVirtIO9P *v9p)
 {
     const char *version = "9P2000.L";
@@ -397,7 +491,8 @@ static void fs_version(QVirtIO9P *v9p)
     char *server_version;
     P9Req *req;
 
-    req = v9fs_tversion(v9p, P9_MAX_SIZE, version);
+    req = v9fs_tversion(v9p, P9_MAX_SIZE, version, P9_NOTAG);
+    v9fs_req_wait_for_reply(req, NULL);
     v9fs_rversion(req, &server_len, &server_version);
 
     g_assert_cmpmem(server_version, server_len, version, strlen(version));
@@ -410,34 +505,31 @@ static void fs_attach(QVirtIO9P *v9p)
     P9Req *req;
 
     fs_version(v9p);
-    req = v9fs_tattach(v9p, 0, getuid());
+    req = v9fs_tattach(v9p, 0, getuid(), 0);
+    v9fs_req_wait_for_reply(req, NULL);
     v9fs_rattach(req, NULL);
 }
 
 static void fs_walk(QVirtIO9P *v9p)
 {
-    char *wnames[P9_MAXWELEM], *paths[P9_MAXWELEM];
-    char *last_path = v9p->test_share;
+    char *wnames[P9_MAXWELEM];
     uint16_t nwqid;
     v9fs_qid *wqid;
     int i;
     P9Req *req;
 
     for (i = 0; i < P9_MAXWELEM; i++) {
-        wnames[i] = g_strdup_printf("%s%d", __func__, i);
-        last_path = paths[i] = g_strdup_printf("%s/%s", last_path, wnames[i]);
-        g_assert(!mkdir(paths[i], 0700));
+        wnames[i] = g_strdup_printf(QTEST_V9FS_SYNTH_WALK_FILE, i);
     }
 
     fs_attach(v9p);
-    req = v9fs_twalk(v9p, 0, 1, P9_MAXWELEM, wnames);
+    req = v9fs_twalk(v9p, 0, 1, P9_MAXWELEM, wnames, 0);
+    v9fs_req_wait_for_reply(req, NULL);
     v9fs_rwalk(req, &nwqid, &wqid);
 
     g_assert_cmpint(nwqid, ==, P9_MAXWELEM);
 
     for (i = 0; i < P9_MAXWELEM; i++) {
-        rmdir(paths[P9_MAXWELEM - i - 1]);
-        g_free(paths[P9_MAXWELEM - i - 1]);
         g_free(wnames[i]);
     }
 
@@ -451,7 +543,8 @@ static void fs_walk_no_slash(QVirtIO9P *v9p)
     uint32_t err;
 
     fs_attach(v9p);
-    req = v9fs_twalk(v9p, 0, 1, 1, wnames);
+    req = v9fs_twalk(v9p, 0, 1, 1, wnames, 0);
+    v9fs_req_wait_for_reply(req, NULL);
     v9fs_rlerror(req, &err);
 
     g_assert_cmpint(err, ==, ENOENT);
@@ -466,10 +559,12 @@ static void fs_walk_dotdot(QVirtIO9P *v9p)
     P9Req *req;
 
     fs_version(v9p);
-    req = v9fs_tattach(v9p, 0, getuid());
+    req = v9fs_tattach(v9p, 0, getuid(), 0);
+    v9fs_req_wait_for_reply(req, NULL);
     v9fs_rattach(req, &root_qid);
 
-    req = v9fs_twalk(v9p, 0, 1, 1, wnames);
+    req = v9fs_twalk(v9p, 0, 1, 1, wnames, 0);
+    v9fs_req_wait_for_reply(req, NULL);
     v9fs_rwalk(req, NULL, &wqid); /* We now we'll get one qid */
 
     g_assert_cmpmem(&root_qid, 13, wqid[0], 13);
@@ -478,6 +573,119 @@ static void fs_walk_dotdot(QVirtIO9P *v9p)
     g_free(wnames[0]);
 }
 
+static void fs_lopen(QVirtIO9P *v9p)
+{
+    char *const wnames[] = { g_strdup(QTEST_V9FS_SYNTH_LOPEN_FILE) };
+    P9Req *req;
+
+    fs_attach(v9p);
+    req = v9fs_twalk(v9p, 0, 1, 1, wnames, 0);
+    v9fs_req_wait_for_reply(req, NULL);
+    v9fs_rwalk(req, NULL, NULL);
+
+    req = v9fs_tlopen(v9p, 1, O_WRONLY, 0);
+    v9fs_req_wait_for_reply(req, NULL);
+    v9fs_rlopen(req, NULL, NULL);
+
+    g_free(wnames[0]);
+}
+
+static void fs_write(QVirtIO9P *v9p)
+{
+    static const uint32_t write_count = P9_MAX_SIZE / 2;
+    char *const wnames[] = { g_strdup(QTEST_V9FS_SYNTH_WRITE_FILE) };
+    char *buf = g_malloc0(write_count);
+    uint32_t count;
+    P9Req *req;
+
+    fs_attach(v9p);
+    req = v9fs_twalk(v9p, 0, 1, 1, wnames, 0);
+    v9fs_req_wait_for_reply(req, NULL);
+    v9fs_rwalk(req, NULL, NULL);
+
+    req = v9fs_tlopen(v9p, 1, O_WRONLY, 0);
+    v9fs_req_wait_for_reply(req, NULL);
+    v9fs_rlopen(req, NULL, NULL);
+
+    req = v9fs_twrite(v9p, 1, 0, write_count, buf, 0);
+    v9fs_req_wait_for_reply(req, NULL);
+    v9fs_rwrite(req, &count);
+    g_assert_cmpint(count, ==, write_count);
+
+    g_free(buf);
+    g_free(wnames[0]);
+}
+
+static void fs_flush_success(QVirtIO9P *v9p)
+{
+    char *const wnames[] = { g_strdup(QTEST_V9FS_SYNTH_FLUSH_FILE) };
+    P9Req *req, *flush_req;
+    uint32_t reply_len;
+    uint8_t should_block;
+
+    fs_attach(v9p);
+    req = v9fs_twalk(v9p, 0, 1, 1, wnames, 0);
+    v9fs_req_wait_for_reply(req, NULL);
+    v9fs_rwalk(req, NULL, NULL);
+
+    req = v9fs_tlopen(v9p, 1, O_WRONLY, 0);
+    v9fs_req_wait_for_reply(req, NULL);
+    v9fs_rlopen(req, NULL, NULL);
+
+    /* This will cause the 9p server to try to write data to the backend,
+     * until the write request gets cancelled.
+     */
+    should_block = 1;
+    req = v9fs_twrite(v9p, 1, 0, sizeof(should_block), &should_block, 0);
+
+    flush_req = v9fs_tflush(v9p, req->tag, 1);
+
+    /* The write request is supposed to be flushed: the server should just
+     * mark the write request as used and reply to the flush request.
+     */
+    v9fs_req_wait_for_reply(req, &reply_len);
+    g_assert_cmpint(reply_len, ==, 0);
+    v9fs_req_free(req);
+    v9fs_rflush(flush_req);
+
+    g_free(wnames[0]);
+}
+
+static void fs_flush_ignored(QVirtIO9P *v9p)
+{
+    char *const wnames[] = { g_strdup(QTEST_V9FS_SYNTH_FLUSH_FILE) };
+    P9Req *req, *flush_req;
+    uint32_t count;
+    uint8_t should_block;
+
+    fs_attach(v9p);
+    req = v9fs_twalk(v9p, 0, 1, 1, wnames, 0);
+    v9fs_req_wait_for_reply(req, NULL);
+    v9fs_rwalk(req, NULL, NULL);
+
+    req = v9fs_tlopen(v9p, 1, O_WRONLY, 0);
+    v9fs_req_wait_for_reply(req, NULL);
+    v9fs_rlopen(req, NULL, NULL);
+
+    /* This will cause the write request to complete right away, before it
+     * could be actually cancelled.
+     */
+    should_block = 0;
+    req = v9fs_twrite(v9p, 1, 0, sizeof(should_block), &should_block, 0);
+
+    flush_req = v9fs_tflush(v9p, req->tag, 1);
+
+    /* The write request is supposed to complete. The server should
+     * reply to the write request and the flush request.
+     */
+    v9fs_req_wait_for_reply(req, NULL);
+    v9fs_rwrite(req, &count);
+    g_assert_cmpint(count, ==, sizeof(should_block));
+    v9fs_rflush(flush_req);
+
+    g_free(wnames[0]);
+}
+
 typedef void (*v9fs_test_fn)(QVirtIO9P *v9p);
 
 static void v9fs_run_pci_test(gconstpointer data)
@@ -507,6 +715,10 @@ int main(int argc, char **argv)
     v9fs_qtest_pci_add("/virtio/9p/pci/fs/walk/no_slash", fs_walk_no_slash);
     v9fs_qtest_pci_add("/virtio/9p/pci/fs/walk/dotdot_from_root",
                        fs_walk_dotdot);
+    v9fs_qtest_pci_add("/virtio/9p/pci/fs/lopen/basic", fs_lopen);
+    v9fs_qtest_pci_add("/virtio/9p/pci/fs/write/basic", fs_write);
+    v9fs_qtest_pci_add("/virtio/9p/pci/fs/flush/success", fs_flush_success);
+    v9fs_qtest_pci_add("/virtio/9p/pci/fs/flush/ignored", fs_flush_ignored);
 
     return g_test_run();
 }

tests/virtio-blk-test.c

@@ -193,7 +193,7 @@ static void test_basic(QVirtioDevice *dev, QGuestAllocator *alloc,
 
     qvirtqueue_kick(dev, vq, free_head);
 
-    qvirtio_wait_used_elem(dev, vq, free_head, QVIRTIO_BLK_TIMEOUT_US);
+    qvirtio_wait_used_elem(dev, vq, free_head, NULL, QVIRTIO_BLK_TIMEOUT_US);
     status = readb(req_addr + 528);
     g_assert_cmpint(status, ==, 0);
 
@@ -215,7 +215,7 @@ static void test_basic(QVirtioDevice *dev, QGuestAllocator *alloc,
 
     qvirtqueue_kick(dev, vq, free_head);
 
-    qvirtio_wait_used_elem(dev, vq, free_head, QVIRTIO_BLK_TIMEOUT_US);
+    qvirtio_wait_used_elem(dev, vq, free_head, NULL, QVIRTIO_BLK_TIMEOUT_US);
     status = readb(req_addr + 528);
     g_assert_cmpint(status, ==, 0);
 
@@ -243,7 +243,8 @@ static void test_basic(QVirtioDevice *dev, QGuestAllocator *alloc,
         qvirtqueue_add(vq, req_addr + 528, 1, true, false);
         qvirtqueue_kick(dev, vq, free_head);
 
-        qvirtio_wait_used_elem(dev, vq, free_head, QVIRTIO_BLK_TIMEOUT_US);
+        qvirtio_wait_used_elem(dev, vq, free_head, NULL,
+                               QVIRTIO_BLK_TIMEOUT_US);
         status = readb(req_addr + 528);
         g_assert_cmpint(status, ==, 0);
 
@@ -264,7 +265,8 @@ static void test_basic(QVirtioDevice *dev, QGuestAllocator *alloc,
 
         qvirtqueue_kick(dev, vq, free_head);
 
-        qvirtio_wait_used_elem(dev, vq, free_head, QVIRTIO_BLK_TIMEOUT_US);
+        qvirtio_wait_used_elem(dev, vq, free_head, NULL,
+                               QVIRTIO_BLK_TIMEOUT_US);
         status = readb(req_addr + 528);
         g_assert_cmpint(status, ==, 0);
 
@@ -345,7 +347,7 @@ static void pci_indirect(void)
     free_head = qvirtqueue_add_indirect(&vqpci->vq, indirect);
     qvirtqueue_kick(&dev->vdev, &vqpci->vq, free_head);
 
-    qvirtio_wait_used_elem(&dev->vdev, &vqpci->vq, free_head,
+    qvirtio_wait_used_elem(&dev->vdev, &vqpci->vq, free_head, NULL,
                            QVIRTIO_BLK_TIMEOUT_US);
     status = readb(req_addr + 528);
     g_assert_cmpint(status, ==, 0);
@@ -370,7 +372,7 @@ static void pci_indirect(void)
     free_head = qvirtqueue_add_indirect(&vqpci->vq, indirect);
     qvirtqueue_kick(&dev->vdev, &vqpci->vq, free_head);
 
-    qvirtio_wait_used_elem(&dev->vdev, &vqpci->vq, free_head,
+    qvirtio_wait_used_elem(&dev->vdev, &vqpci->vq, free_head, NULL,
                            QVIRTIO_BLK_TIMEOUT_US);
     status = readb(req_addr + 528);
     g_assert_cmpint(status, ==, 0);
@@ -481,7 +483,7 @@ static void pci_msix(void)
     qvirtqueue_add(&vqpci->vq, req_addr + 528, 1, true, false);
     qvirtqueue_kick(&dev->vdev, &vqpci->vq, free_head);
 
-    qvirtio_wait_used_elem(&dev->vdev, &vqpci->vq, free_head,
+    qvirtio_wait_used_elem(&dev->vdev, &vqpci->vq, free_head, NULL,
                            QVIRTIO_BLK_TIMEOUT_US);
 
     status = readb(req_addr + 528);
@@ -506,7 +508,7 @@ static void pci_msix(void)
     qvirtqueue_kick(&dev->vdev, &vqpci->vq, free_head);
 
 
-    qvirtio_wait_used_elem(&dev->vdev, &vqpci->vq, free_head,
+    qvirtio_wait_used_elem(&dev->vdev, &vqpci->vq, free_head, NULL,
                            QVIRTIO_BLK_TIMEOUT_US);
 
     status = readb(req_addr + 528);
@@ -580,7 +582,7 @@ static void pci_idx(void)
     qvirtqueue_add(&vqpci->vq, req_addr + 528, 1, true, false);
     qvirtqueue_kick(&dev->vdev, &vqpci->vq, free_head);
 
-    qvirtio_wait_used_elem(&dev->vdev, &vqpci->vq, free_head,
+    qvirtio_wait_used_elem(&dev->vdev, &vqpci->vq, free_head, NULL,
                            QVIRTIO_BLK_TIMEOUT_US);
 
     /* Write request */
@@ -627,9 +629,9 @@ static void pci_idx(void)
     qvirtqueue_kick(&dev->vdev, &vqpci->vq, free_head);
 
     /* We get just one notification for both requests */
-    qvirtio_wait_used_elem(&dev->vdev, &vqpci->vq, write_head,
+    qvirtio_wait_used_elem(&dev->vdev, &vqpci->vq, write_head, NULL,
                            QVIRTIO_BLK_TIMEOUT_US);
-    g_assert(qvirtqueue_get_buf(&vqpci->vq, &desc_idx));
+    g_assert(qvirtqueue_get_buf(&vqpci->vq, &desc_idx, NULL));
     g_assert_cmpint(desc_idx, ==, free_head);
 
     status = readb(req_addr + 528);

tests/virtio-net-test.c

@@ -108,7 +108,7 @@ static void rx_test(QVirtioDevice *dev,
     ret = iov_send(socket, iov, 2, 0, sizeof(len) + sizeof(test));
     g_assert_cmpint(ret, ==, sizeof(test) + sizeof(len));
 
-    qvirtio_wait_used_elem(dev, vq, free_head, QVIRTIO_NET_TIMEOUT_US);
+    qvirtio_wait_used_elem(dev, vq, free_head, NULL, QVIRTIO_NET_TIMEOUT_US);
     memread(req_addr + VNET_HDR_SIZE, buffer, sizeof(test));
     g_assert_cmpstr(buffer, ==, "TEST");
 
@@ -131,7 +131,7 @@ static void tx_test(QVirtioDevice *dev,
     free_head = qvirtqueue_add(vq, req_addr, 64, false, false);
     qvirtqueue_kick(dev, vq, free_head);
 
-    qvirtio_wait_used_elem(dev, vq, free_head, QVIRTIO_NET_TIMEOUT_US);
+    qvirtio_wait_used_elem(dev, vq, free_head, NULL, QVIRTIO_NET_TIMEOUT_US);
     guest_free(alloc, req_addr);
 
     ret = qemu_recv(socket, &len, sizeof(len), 0);
@@ -182,7 +182,7 @@ static void rx_stop_cont_test(QVirtioDevice *dev,
     rsp = qmp("{ 'execute' : 'cont'}");
     QDECREF(rsp);
 
-    qvirtio_wait_used_elem(dev, vq, free_head, QVIRTIO_NET_TIMEOUT_US);
+    qvirtio_wait_used_elem(dev, vq, free_head, NULL, QVIRTIO_NET_TIMEOUT_US);
     memread(req_addr + VNET_HDR_SIZE, buffer, sizeof(test));
     g_assert_cmpstr(buffer, ==, "TEST");
 

tests/virtio-scsi-test.c

@@ -121,7 +121,8 @@ static uint8_t virtio_scsi_do_command(QVirtIOSCSI *vs, const uint8_t *cdb,
     }
 
     qvirtqueue_kick(vs->dev, vq, free_head);
-    qvirtio_wait_used_elem(vs->dev, vq, free_head, QVIRTIO_SCSI_TIMEOUT_US);
+    qvirtio_wait_used_elem(vs->dev, vq, free_head, NULL,
+                           QVIRTIO_SCSI_TIMEOUT_US);
 
     response = readb(resp_addr +
                      offsetof(struct virtio_scsi_cmd_resp, response));