libctf: write CTF files to memory, and CTF archives to fds

Before now, we've been able to write CTF files to gzFile descriptors or
fds, and CTF archives to named files only.

Make this a bit less irregular by allowing CTF archives to be written
to fds with the new function ctf_arc_write_fd: also allow CTF
files to be written to a new memory buffer via ctf_write_mem.

(It would be nice to complete things by adding a new function to write
CTF archives to memory, but this is too difficult to do given the short
time the linker is expected to be writing them out: we will transition
to a better format in format v4, though we will always support reading
CTF archives that are stored in .ctf sections.)

include/
	* ctf-api.h (ctf_arc_write_fd): New.
	(ctf_write_mem): Likewise.
	(ctf_gzwrite): Spacing fix.

libctf/
	* ctf-archive.c (ctf_arc_write): Split off, and reimplement in terms
	of...
	(ctf_arc_write_fd): ... this new function.
	* ctf-create.c (ctf_write_mem): New.

include/ChangeLog

@@ -1,3 +1,9 @@
+2019-07-13  Nick Alcock  <nick.alcock@oracle.com>
+
+	* ctf-api.h (ctf_arc_write_fd): New.
+	(ctf_write_mem): Likewise.
+	(ctf_gzwrite): Spacing fix.
+
 2019-07-13  Nick Alcock  <nick.alcock@oracle.com>
 
 	* ctf.h (CTF_SET_STID): New.

include/ctf-api.h

@@ -258,6 +258,8 @@ extern void ctf_file_close (ctf_file_t *);
 
 extern int ctf_arc_write (const char *, ctf_file_t **, size_t,
 			  const char **, size_t);
+extern int ctf_arc_write_fd (int, ctf_file_t **, size_t, const char **,
+			     size_t);
 
 extern const char *ctf_cuname (ctf_file_t *);
 extern void ctf_cuname_set (ctf_file_t *, const char *);
@@ -379,8 +381,9 @@ extern ctf_snapshot_id_t ctf_snapshot (ctf_file_t *);
 extern int ctf_rollback (ctf_file_t *, ctf_snapshot_id_t);
 extern int ctf_discard (ctf_file_t *);
 extern int ctf_write (ctf_file_t *, int);
-extern int ctf_gzwrite (ctf_file_t * fp, gzFile fd);
+extern int ctf_gzwrite (ctf_file_t *fp, gzFile fd);
 extern int ctf_compress_write (ctf_file_t * fp, int fd);
+extern unsigned char *ctf_write_mem (ctf_file_t *, size_t *, size_t threshold);
 
 extern void ctf_setdebug (int debug);
 extern int ctf_getdebug (void);

libctf/ChangeLog

@@ -1,3 +1,10 @@
+2019-07-13  Nick Alcock  <nick.alcock@oracle.com>
+
+	* ctf-archive.c (ctf_arc_write): Split off, and reimplement in terms
+	of...
+	(ctf_arc_write_fd): ... this new function.
+	* ctf-create.c (ctf_write_mem): New.
+
 2019-07-13  Nick Alcock  <nick.alcock@oracle.com>
 
 	* ctf-impl.h (ctf_str_atom_t) <csa_offset>: New field.

libctf/ctf-archive.c

@@ -47,17 +47,17 @@ static int arc_mmap_unmap (void *header, size_t headersz, const char **errmsg);
 /* bsearch() internal state.  */
 static __thread char *search_nametbl;
 
-/* Write out a CTF archive.  The entries in CTF_FILES are referenced by name:
-   the names are passed in the names array, which must have CTF_FILES entries.
+/* Write out a CTF archive to the start of the file referenced by the passed-in
+   fd.  The entries in CTF_FILES are referenced by name: the names are passed in
+   the names array, which must have CTF_FILES entries.
 
    Returns 0 on success, or an errno, or an ECTF_* value.  */
 int
-ctf_arc_write (const char *file, ctf_file_t ** ctf_files, size_t ctf_file_cnt,
-	       const char **names, size_t threshold)
+ctf_arc_write_fd (int fd, ctf_file_t **ctf_files, size_t ctf_file_cnt,
+		  const char **names, size_t threshold)
 {
   const char *errmsg;
   struct ctf_archive *archdr;
-  int fd;
   size_t i;
   char dummy = 0;
   size_t headersz;
@@ -68,15 +68,9 @@ ctf_arc_write (const char *file, ctf_file_t ** ctf_files, size_t ctf_file_cnt,
   off_t nameoffs;
   struct ctf_archive_modent *modent;
 
-  ctf_dprintf ("Writing archive %s with %lu files\n", file,
+  ctf_dprintf ("Writing CTF archive with %lu files\n",
 	       (unsigned long) ctf_file_cnt);
 
-  if ((fd = open (file, O_RDWR | O_CREAT | O_TRUNC | O_CLOEXEC, 0666)) < 0)
-    {
-      errmsg = "ctf_arc_write(): cannot create %s: %s\n";
-      goto err;
-    }
-
   /* Figure out the size of the mmap()ed header, including the
      ctf_archive_modent array.  We assume that all of this needs no
      padding: a likely assumption, given that it's all made up of
@@ -91,20 +85,20 @@ ctf_arc_write (const char *file, ctf_file_t ** ctf_files, size_t ctf_file_cnt,
   ctf_startoffs = headersz;
   if (lseek (fd, ctf_startoffs - 1, SEEK_SET) < 0)
     {
-      errmsg = "ctf_arc_write(): cannot extend file while writing %s: %s\n";
-      goto err_close;
+      errmsg = "ctf_arc_write(): cannot extend file while writing: %s\n";
+      goto err;
     }
 
   if (write (fd, &dummy, 1) < 0)
     {
-      errmsg = "ctf_arc_write(): cannot extend file while writing %s: %s\n";
-      goto err_close;
+      errmsg = "ctf_arc_write(): cannot extend file while writing: %s\n";
+      goto err;
     }
 
   if ((archdr = arc_mmap_header (fd, headersz)) == NULL)
     {
-      errmsg = "ctf_arc_write(): Cannot mmap() %s: %s\n";
-      goto err_close;
+      errmsg = "ctf_arc_write(): Cannot mmap(): %s\n";
+      goto err;
     }
 
   /* Fill in everything we can, which is everything other than the name
@@ -137,7 +131,7 @@ ctf_arc_write (const char *file, ctf_file_t ** ctf_files, size_t ctf_file_cnt,
   nametbl = malloc (namesz);
   if (nametbl == NULL)
     {
-      errmsg = "Error writing named CTF to %s: %s\n";
+      errmsg = "Error writing named CTF to archive: %s\n";
       goto err_unmap;
     }
 
@@ -154,12 +148,12 @@ ctf_arc_write (const char *file, ctf_file_t ** ctf_files, size_t ctf_file_cnt,
       if ((off < 0) && (off > -ECTF_BASE))
 	{
 	  errmsg = "ctf_arc_write(): Cannot determine file "
-	    "position while writing %s: %s";
+	    "position while writing to archive: %s";
 	  goto err_free;
 	}
       if (off < 0)
 	{
-	  errmsg = "ctf_arc_write(): Cannot write CTF file to %s: %s\n";
+	  errmsg = "ctf_arc_write(): Cannot write CTF file to archive: %s\n";
 	  errno = off * -1;
 	  goto err_free;
 	}
@@ -181,7 +175,7 @@ ctf_arc_write (const char *file, ctf_file_t ** ctf_files, size_t ctf_file_cnt,
   if ((nameoffs = lseek (fd, 0, SEEK_CUR)) < 0)
     {
       errmsg = "ctf_arc_write(): Cannot get current file position "
-	"in %s: %s\n";
+	"in archive: %s\n";
       goto err_free;
     }
   archdr->ctfa_names = htole64 (nameoffs);
@@ -191,7 +185,7 @@ ctf_arc_write (const char *file, ctf_file_t ** ctf_files, size_t ctf_file_cnt,
       ssize_t len;
       if ((len = write (fd, np, namesz)) < 0)
 	{
-	  errmsg = "ctf_arc_write(): Cannot write name table in %s: %s\n";
+	  errmsg = "ctf_arc_write(): Cannot write name table to archive: %s\n";
 	  goto err_free;
 	}
       namesz -= len;
@@ -202,29 +196,64 @@ ctf_arc_write (const char *file, ctf_file_t ** ctf_files, size_t ctf_file_cnt,
   if (arc_mmap_writeout (fd, archdr, headersz, &errmsg) < 0)
     goto err_unmap;
   if (arc_mmap_unmap (archdr, headersz, &errmsg) < 0)
-    goto err_unlink;
-  if (close (fd) < 0)
-    {
-      errmsg = "ctf_arc_write(): Cannot close after writing to %s: %s\n";
-      goto err_unlink;
-    }
-
+    goto err;
   return 0;
 
 err_free:
   free (nametbl);
 err_unmap:
   arc_mmap_unmap (archdr, headersz, NULL);
-err_close:
-  close (fd);
-err_unlink:
-  unlink (file);
 err:
-  ctf_dprintf (errmsg, file, errno < ECTF_BASE ? strerror (errno) :
+  ctf_dprintf (errmsg, errno < ECTF_BASE ? strerror (errno) :
 	       ctf_errmsg (errno));
   return errno;
 }
 
+/* Write out a CTF archive.  The entries in CTF_FILES are referenced by name:
+   the names are passed in the names array, which must have CTF_FILES entries.
+
+   If the filename is NULL, create a temporary file and return a pointer to it.
+
+   Returns 0 on success, or an errno, or an ECTF_* value.  */
+int
+ctf_arc_write (const char *file, ctf_file_t ** ctf_files, size_t ctf_file_cnt,
+	       const char **names, size_t threshold)
+{
+  int err;
+  int fd;
+
+  if ((fd = open (file, O_RDWR | O_CREAT | O_TRUNC | O_CLOEXEC, 0666)) < 0)
+    {
+      ctf_dprintf ("ctf_arc_write(): cannot create %s: %s\n", file,
+		   strerror (errno));
+      return errno;
+    }
+
+  err = ctf_arc_write_fd (fd, ctf_files, ctf_file_cnt, names, threshold);
+  if (err)
+    goto err;
+
+  if ((err = close (fd)) < 0)
+    {
+      ctf_dprintf ("ctf_arc_write(): Cannot close after writing to archive: "
+		   "%s\n", strerror (errno));
+      goto err_close;
+    }
+
+ err:
+  close (fd);
+  if (err < 0)
+    unlink (file);
+
+  return err;
+
+ err_close:
+  if (err < 0)
+    unlink (file);
+
+  return err;
+}
+
 /* Write one CTF file out.  Return the file position of the written file (or
    rather, of the file-size uint64_t that precedes it): negative return is a
    negative errno or ctf_errno value.  On error, the file position may no longer

libctf/ctf-create.c

@@ -2029,6 +2029,57 @@ ret:
   return err;
 }
 
+/* Optionally compress the specified CTF data stream and return it as a new
+   dynamically-allocated string.  */
+unsigned char *
+ctf_write_mem (ctf_file_t *fp, size_t *size, size_t threshold)
+{
+  unsigned char *buf;
+  unsigned char *bp;
+  ctf_header_t *hp;
+  ssize_t header_len = sizeof (ctf_header_t);
+  ssize_t compress_len;
+  size_t max_compress_len = compressBound (fp->ctf_size);
+  int rc;
+
+  if (fp->ctf_size < threshold)
+    max_compress_len = fp->ctf_size;
+  if ((buf = malloc (max_compress_len
+		     + sizeof (struct ctf_header))) == NULL)
+    {
+      ctf_set_errno (fp, ENOMEM);
+      return NULL;
+    }
+
+  hp = (ctf_header_t *) buf;
+  memcpy (hp, fp->ctf_header, header_len);
+  bp = buf + sizeof (struct ctf_header);
+  *size = sizeof (struct ctf_header);
+
+  compress_len = max_compress_len;
+
+  if (fp->ctf_size < threshold)
+    {
+      hp->cth_flags &= ~CTF_F_COMPRESS;
+      memcpy (bp, fp->ctf_buf, fp->ctf_size);
+      *size += fp->ctf_size;
+    }
+  else
+    {
+      hp->cth_flags |= CTF_F_COMPRESS;
+      if ((rc = compress (bp, (uLongf *) &compress_len,
+			  fp->ctf_buf, fp->ctf_size)) != Z_OK)
+	{
+	  ctf_dprintf ("zlib deflate err: %s\n", zError (rc));
+	  ctf_set_errno (fp, ECTF_COMPRESS);
+	  ctf_free (buf);
+	  return NULL;
+	}
+      *size += compress_len;
+    }
+  return buf;
+}
+
 /* Write the uncompressed CTF data stream to the specified file descriptor.  */
 int
 ctf_write (ctf_file_t *fp, int fd)