libctf, dedup: add deduplicator

This adds the core deduplicator that the ctf_link machinery calls
(possibly repeatedly) to link the CTF sections: it takes an array
of input ctf_file_t's and another array that indicates which entries in
the input array are parents of which other entries, and returns an array
of outputs.  The first output is always the ctf_file_t on which
ctf_link/ctf_dedup/etc was called: the other outputs are child dicts
that have the first output as their parent.

include/
	* ctf-api.h (CTF_LINK_SHARE_DUPLICATED): No longer unimplemented.
libctf/
	* ctf-impl.h (ctf_type_id_key): New, the key in the
	cd_id_to_file_t.
	(ctf_dedup): New, core deduplicator state.
	(ctf_file_t) <ctf_dedup>: New.
	<ctf_dedup_atoms>: New.
	<ctf_dedup_atoms_alloc>: New.
	(ctf_hash_type_id_key): New prototype.
	(ctf_hash_eq_type_id_key): Likewise.
	(ctf_dedup_atoms_init): Likewise.
	* ctf-hash.c (ctf_hash_eq_type_id_key): New.
	(ctf_dedup_atoms_init): Likewise.
	* ctf-create.c (ctf_serialize): Adjusted.
	(ctf_add_encoded): No longer static.
	(ctf_add_reftype): Likewise.
	* ctf-open.c (ctf_file_close): Destroy the
	ctf_dedup_atoms_alloc.
	* ctf-dedup.c: New file.
        * ctf-decls.h [!HAVE_DECL_STPCPY]: Add prototype.
	* configure.ac: Check for stpcpy.
	* Makefile.am: Add it.
	* Makefile.in: Regenerate.
        * config.h.in: Regenerate.
        * configure: Regenerate.

include/ChangeLog

@@ -1,3 +1,7 @@
+2020-07-22  Nick Alcock  <nick.alcock@oracle.com>
+
+	* ctf-api.h (CTF_LINK_SHARE_DUPLICATED): No longer unimplemented.
+
 2020-07-22  Nick Alcock  <nick.alcock@oracle.com>
 
 	* ctf-api.h (ctf_link_variable_filter_t): New.

include/ctf-api.h

@@ -83,7 +83,7 @@ typedef struct ctf_link_sym
 /* Share all types that are not in conflict.  The default.  */
 #define CTF_LINK_SHARE_UNCONFLICTED 0x0
 
-/* Share only types that are used by multiple inputs.  Not implemented yet.  */
+/* Share only types that are used by multiple inputs.  */
 #define CTF_LINK_SHARE_DUPLICATED 0x1
 
 /* Create empty outputs for all registered CU mappings even if no types are

libctf/ChangeLog

@@ -1,3 +1,29 @@
+2020-07-22  Nick Alcock  <nick.alcock@oracle.com>
+
+	* ctf-impl.h (ctf_type_id_key): New, the key in the
+	cd_id_to_file_t.
+	(ctf_dedup): New, core deduplicator state.
+	(ctf_file_t) <ctf_dedup>: New.
+	<ctf_dedup_atoms>: New.
+	<ctf_dedup_atoms_alloc>: New.
+	(ctf_hash_type_id_key): New prototype.
+	(ctf_hash_eq_type_id_key): Likewise.
+	(ctf_dedup_atoms_init): Likewise.
+	* ctf-hash.c (ctf_hash_eq_type_id_key): New.
+	(ctf_dedup_atoms_init): Likewise.
+	* ctf-create.c (ctf_serialize): Adjusted.
+	(ctf_add_encoded): No longer static.
+	(ctf_add_reftype): Likewise.
+	* ctf-open.c (ctf_file_close): Destroy the
+	ctf_dedup_atoms_alloc.
+	* ctf-dedup.c: New file.
+        * ctf-decls.h [!HAVE_DECL_STPCPY]: Add prototype.
+	* configure.ac: Check for stpcpy.
+	* Makefile.am: Add it.
+	* Makefile.in: Regenerate.
+        * config.h.in: Regenerate.
+        * configure: Regenerate.
+
 2020-07-22  Nick Alcock  <nick.alcock@oracle.com>
 
 	* configure.ac: Add --enable-libctf-hash-debugging.

libctf/Makefile.am

@@ -43,8 +43,9 @@ libctf_nobfd_la_LIBADD = @SHARED_LIBADD@ $(ZLIB)
 libctf_nobfd_la_LDFLAGS = -version-info 0:0:0 @SHARED_LDFLAGS@ @VERSION_FLAGS@
 libctf_nobfd_la_CPPFLAGS = $(AM_CPPFLAGS) -DNOBFD=1
 libctf_nobfd_la_SOURCES = ctf-archive.c ctf-dump.c ctf-create.c ctf-decl.c ctf-error.c \
-			  ctf-hash.c ctf-labels.c ctf-link.c ctf-lookup.c ctf-open.c \
+			  ctf-hash.c ctf-labels.c ctf-dedup.c ctf-link.c ctf-lookup.c \
-			  ctf-sha1.c ctf-string.c ctf-subr.c ctf-types.c ctf-util.c
+			  ctf-open.c ctf-sha1.c ctf-string.c ctf-subr.c ctf-types.c \
+			  ctf-util.c
 if NEED_CTF_QSORT_R
 libctf_nobfd_la_SOURCES += ctf-qsort_r.c
 endif

libctf/Makefile.in

@@ -166,18 +166,18 @@ am__DEPENDENCIES_1 =
 libctf_nobfd_la_DEPENDENCIES = $(am__DEPENDENCIES_1)
 am__libctf_nobfd_la_SOURCES_DIST = ctf-archive.c ctf-dump.c \
 	ctf-create.c ctf-decl.c ctf-error.c ctf-hash.c ctf-labels.c \
-	ctf-link.c ctf-lookup.c ctf-open.c ctf-sha1.c ctf-string.c \
+	ctf-dedup.c ctf-link.c ctf-lookup.c ctf-open.c ctf-sha1.c \
-	ctf-subr.c ctf-types.c ctf-util.c ctf-qsort_r.c
+	ctf-string.c ctf-subr.c ctf-types.c ctf-util.c ctf-qsort_r.c
 @NEED_CTF_QSORT_R_TRUE@am__objects_1 = libctf_nobfd_la-ctf-qsort_r.lo
 am_libctf_nobfd_la_OBJECTS = libctf_nobfd_la-ctf-archive.lo \
 	libctf_nobfd_la-ctf-dump.lo libctf_nobfd_la-ctf-create.lo \
 	libctf_nobfd_la-ctf-decl.lo libctf_nobfd_la-ctf-error.lo \
 	libctf_nobfd_la-ctf-hash.lo libctf_nobfd_la-ctf-labels.lo \
-	libctf_nobfd_la-ctf-link.lo libctf_nobfd_la-ctf-lookup.lo \
+	libctf_nobfd_la-ctf-dedup.lo libctf_nobfd_la-ctf-link.lo \
-	libctf_nobfd_la-ctf-open.lo libctf_nobfd_la-ctf-sha1.lo \
+	libctf_nobfd_la-ctf-lookup.lo libctf_nobfd_la-ctf-open.lo \
-	libctf_nobfd_la-ctf-string.lo libctf_nobfd_la-ctf-subr.lo \
+	libctf_nobfd_la-ctf-sha1.lo libctf_nobfd_la-ctf-string.lo \
-	libctf_nobfd_la-ctf-types.lo libctf_nobfd_la-ctf-util.lo \
+	libctf_nobfd_la-ctf-subr.lo libctf_nobfd_la-ctf-types.lo \
-	$(am__objects_1)
+	libctf_nobfd_la-ctf-util.lo $(am__objects_1)
 libctf_nobfd_la_OBJECTS = $(am_libctf_nobfd_la_OBJECTS)
 AM_V_lt = $(am__v_lt_@AM_V@)
 am__v_lt_ = $(am__v_lt_@AM_DEFAULT_V@)
@@ -191,18 +191,18 @@ libctf_nobfd_la_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CC \
 @INSTALL_LIBBFD_TRUE@am_libctf_nobfd_la_rpath = -rpath $(libdir)
 am__DEPENDENCIES_2 = $(am__DEPENDENCIES_1)
 am__libctf_la_SOURCES_DIST = ctf-archive.c ctf-dump.c ctf-create.c \
-	ctf-decl.c ctf-error.c ctf-hash.c ctf-labels.c ctf-link.c \
+	ctf-decl.c ctf-error.c ctf-hash.c ctf-labels.c ctf-dedup.c \
-	ctf-lookup.c ctf-open.c ctf-sha1.c ctf-string.c ctf-subr.c \
+	ctf-link.c ctf-lookup.c ctf-open.c ctf-sha1.c ctf-string.c \
-	ctf-types.c ctf-util.c ctf-qsort_r.c ctf-open-bfd.c
+	ctf-subr.c ctf-types.c ctf-util.c ctf-qsort_r.c ctf-open-bfd.c
 @NEED_CTF_QSORT_R_TRUE@am__objects_2 = libctf_la-ctf-qsort_r.lo
 am__objects_3 = libctf_la-ctf-archive.lo libctf_la-ctf-dump.lo \
 	libctf_la-ctf-create.lo libctf_la-ctf-decl.lo \
 	libctf_la-ctf-error.lo libctf_la-ctf-hash.lo \
-	libctf_la-ctf-labels.lo libctf_la-ctf-link.lo \
+	libctf_la-ctf-labels.lo libctf_la-ctf-dedup.lo \
-	libctf_la-ctf-lookup.lo libctf_la-ctf-open.lo \
+	libctf_la-ctf-link.lo libctf_la-ctf-lookup.lo \
-	libctf_la-ctf-sha1.lo libctf_la-ctf-string.lo \
+	libctf_la-ctf-open.lo libctf_la-ctf-sha1.lo \
-	libctf_la-ctf-subr.lo libctf_la-ctf-types.lo \
+	libctf_la-ctf-string.lo libctf_la-ctf-subr.lo \
-	libctf_la-ctf-util.lo $(am__objects_2)
+	libctf_la-ctf-types.lo libctf_la-ctf-util.lo $(am__objects_2)
 am_libctf_la_OBJECTS = $(am__objects_3) libctf_la-ctf-open-bfd.lo
 libctf_la_OBJECTS = $(am_libctf_la_OBJECTS)
 libctf_la_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) \
@@ -457,9 +457,9 @@ libctf_nobfd_la_LIBADD = @SHARED_LIBADD@ $(ZLIB)
 libctf_nobfd_la_LDFLAGS = -version-info 0:0:0 @SHARED_LDFLAGS@ @VERSION_FLAGS@
 libctf_nobfd_la_CPPFLAGS = $(AM_CPPFLAGS) -DNOBFD=1
 libctf_nobfd_la_SOURCES = ctf-archive.c ctf-dump.c ctf-create.c \
-	ctf-decl.c ctf-error.c ctf-hash.c ctf-labels.c ctf-link.c \
+	ctf-decl.c ctf-error.c ctf-hash.c ctf-labels.c ctf-dedup.c \
-	ctf-lookup.c ctf-open.c ctf-sha1.c ctf-string.c ctf-subr.c \
+	ctf-link.c ctf-lookup.c ctf-open.c ctf-sha1.c ctf-string.c \
-	ctf-types.c ctf-util.c $(am__append_1)
+	ctf-subr.c ctf-types.c ctf-util.c $(am__append_1)
 libctf_la_LIBADD = @BFD_LIBADD@ $(libctf_nobfd_la_LIBADD)
 libctf_la_CPPFLAGS = $(AM_CPPFLAGS) -DNOBFD=0
 libctf_la_DEPENDENCIES = @BFD_DEPENDENCIES@
@@ -581,6 +581,7 @@ distclean-compile:
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libctf_la-ctf-archive.Plo@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libctf_la-ctf-create.Plo@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libctf_la-ctf-decl.Plo@am__quote@
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libctf_la-ctf-dedup.Plo@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libctf_la-ctf-dump.Plo@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libctf_la-ctf-error.Plo@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libctf_la-ctf-hash.Plo@am__quote@
@@ -598,6 +599,7 @@ distclean-compile:
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libctf_nobfd_la-ctf-archive.Plo@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libctf_nobfd_la-ctf-create.Plo@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libctf_nobfd_la-ctf-decl.Plo@am__quote@
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libctf_nobfd_la-ctf-dedup.Plo@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libctf_nobfd_la-ctf-dump.Plo@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libctf_nobfd_la-ctf-error.Plo@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/libctf_nobfd_la-ctf-hash.Plo@am__quote@
@@ -682,6 +684,13 @@ libctf_nobfd_la-ctf-labels.lo: ctf-labels.c
 @AMDEP_TRUE@@am__fastdepCC_FALSE@	DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
 @am__fastdepCC_FALSE@	$(AM_V_CC@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libctf_nobfd_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o libctf_nobfd_la-ctf-labels.lo `test -f 'ctf-labels.c' || echo '$(srcdir)/'`ctf-labels.c
 
+libctf_nobfd_la-ctf-dedup.lo: ctf-dedup.c
+@am__fastdepCC_TRUE@	$(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libctf_nobfd_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT libctf_nobfd_la-ctf-dedup.lo -MD -MP -MF $(DEPDIR)/libctf_nobfd_la-ctf-dedup.Tpo -c -o libctf_nobfd_la-ctf-dedup.lo `test -f 'ctf-dedup.c' || echo '$(srcdir)/'`ctf-dedup.c
+@am__fastdepCC_TRUE@	$(AM_V_at)$(am__mv) $(DEPDIR)/libctf_nobfd_la-ctf-dedup.Tpo $(DEPDIR)/libctf_nobfd_la-ctf-dedup.Plo
+@AMDEP_TRUE@@am__fastdepCC_FALSE@	$(AM_V_CC)source='ctf-dedup.c' object='libctf_nobfd_la-ctf-dedup.lo' libtool=yes @AMDEPBACKSLASH@
+@AMDEP_TRUE@@am__fastdepCC_FALSE@	DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+@am__fastdepCC_FALSE@	$(AM_V_CC@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libctf_nobfd_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o libctf_nobfd_la-ctf-dedup.lo `test -f 'ctf-dedup.c' || echo '$(srcdir)/'`ctf-dedup.c
+
 libctf_nobfd_la-ctf-link.lo: ctf-link.c
 @am__fastdepCC_TRUE@	$(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libctf_nobfd_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT libctf_nobfd_la-ctf-link.lo -MD -MP -MF $(DEPDIR)/libctf_nobfd_la-ctf-link.Tpo -c -o libctf_nobfd_la-ctf-link.lo `test -f 'ctf-link.c' || echo '$(srcdir)/'`ctf-link.c
 @am__fastdepCC_TRUE@	$(AM_V_at)$(am__mv) $(DEPDIR)/libctf_nobfd_la-ctf-link.Tpo $(DEPDIR)/libctf_nobfd_la-ctf-link.Plo
@@ -794,6 +803,13 @@ libctf_la-ctf-labels.lo: ctf-labels.c
 @AMDEP_TRUE@@am__fastdepCC_FALSE@	DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
 @am__fastdepCC_FALSE@	$(AM_V_CC@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libctf_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o libctf_la-ctf-labels.lo `test -f 'ctf-labels.c' || echo '$(srcdir)/'`ctf-labels.c
 
+libctf_la-ctf-dedup.lo: ctf-dedup.c
+@am__fastdepCC_TRUE@	$(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libctf_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT libctf_la-ctf-dedup.lo -MD -MP -MF $(DEPDIR)/libctf_la-ctf-dedup.Tpo -c -o libctf_la-ctf-dedup.lo `test -f 'ctf-dedup.c' || echo '$(srcdir)/'`ctf-dedup.c
+@am__fastdepCC_TRUE@	$(AM_V_at)$(am__mv) $(DEPDIR)/libctf_la-ctf-dedup.Tpo $(DEPDIR)/libctf_la-ctf-dedup.Plo
+@AMDEP_TRUE@@am__fastdepCC_FALSE@	$(AM_V_CC)source='ctf-dedup.c' object='libctf_la-ctf-dedup.lo' libtool=yes @AMDEPBACKSLASH@
+@AMDEP_TRUE@@am__fastdepCC_FALSE@	DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+@am__fastdepCC_FALSE@	$(AM_V_CC@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libctf_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o libctf_la-ctf-dedup.lo `test -f 'ctf-dedup.c' || echo '$(srcdir)/'`ctf-dedup.c
+
 libctf_la-ctf-link.lo: ctf-link.c
 @am__fastdepCC_TRUE@	$(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libctf_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT libctf_la-ctf-link.lo -MD -MP -MF $(DEPDIR)/libctf_la-ctf-link.Tpo -c -o libctf_la-ctf-link.lo `test -f 'ctf-link.c' || echo '$(srcdir)/'`ctf-link.c
 @am__fastdepCC_TRUE@	$(AM_V_at)$(am__mv) $(DEPDIR)/libctf_la-ctf-link.Tpo $(DEPDIR)/libctf_la-ctf-link.Plo

libctf/config.h.in

@@ -32,6 +32,10 @@
    don't. */
 #undef HAVE_DECL_BSWAP_64
 
+/* Define to 1 if you have the declaration of `stpcpy', and to 0 if you don't.
+   */
+#undef HAVE_DECL_STPCPY
+
 /* Define to 1 if you have the declaration of `vasprintf', and to 0 if you
    don't. */
 #undef HAVE_DECL_VASPRINTF

libctf/configure

@@ -13186,6 +13186,16 @@ fi
 cat >>confdefs.h <<_ACEOF
 #define HAVE_DECL_VASPRINTF $ac_have_decl
 _ACEOF
+ac_fn_c_check_decl "$LINENO" "stpcpy" "ac_cv_have_decl_stpcpy" "$ac_includes_default"
+if test "x$ac_cv_have_decl_stpcpy" = xyes; then :
+  ac_have_decl=1
+else
+  ac_have_decl=0
+fi
+
+cat >>confdefs.h <<_ACEOF
+#define HAVE_DECL_STPCPY $ac_have_decl
+_ACEOF
 
 
 

libctf/configure.ac

@@ -108,7 +108,7 @@ AC_CHECK_FUNCS(pread)
 
 dnl Check for bswap_{16,32,64}
 AC_CHECK_DECLS([bswap_16, bswap_32, bswap_64], [], [], [[#include <byteswap.h>]])
-AC_CHECK_DECLS([asprintf, vasprintf])
+AC_CHECK_DECLS([asprintf, vasprintf, stpcpy])
 
 dnl Check for qsort_r.  (Taken from gnulib.)
 AC_CHECK_FUNCS_ONCE([qsort_r])

libctf/ctf-create.c

@@ -546,6 +546,9 @@ ctf_serialize (ctf_file_t *fp)
   nfp->ctf_link_variable_filter = fp->ctf_link_variable_filter;
   nfp->ctf_link_variable_filter_arg = fp->ctf_link_variable_filter_arg;
   nfp->ctf_link_flags = fp->ctf_link_flags;
+  nfp->ctf_dedup_atoms = fp->ctf_dedup_atoms;
+  nfp->ctf_dedup_atoms_alloc = fp->ctf_dedup_atoms_alloc;
+  memcpy (&nfp->ctf_dedup, &fp->ctf_dedup, sizeof (fp->ctf_dedup));
 
   nfp->ctf_snapshot_lu = fp->ctf_snapshots;
 
@@ -571,11 +574,14 @@ ctf_serialize (ctf_file_t *fp)
   fp->ctf_link_in_cu_mapping = NULL;
   fp->ctf_link_out_cu_mapping = NULL;
   fp->ctf_link_type_mapping = NULL;
+  fp->ctf_dedup_atoms = NULL;
+  fp->ctf_dedup_atoms_alloc = NULL;
   fp->ctf_parent_unreffed = 1;
 
   fp->ctf_dvhash = NULL;
   memset (&fp->ctf_dvdefs, 0, sizeof (ctf_list_t));
   memset (fp->ctf_lookups, 0, sizeof (fp->ctf_lookups));
+  memset (&fp->ctf_dedup, 0, sizeof (fp->ctf_dedup));
   fp->ctf_structs.ctn_writable = NULL;
   fp->ctf_unions.ctn_writable = NULL;
   fp->ctf_enums.ctn_writable = NULL;
@@ -878,7 +884,7 @@ clp2 (size_t x)
   return (x + 1);
 }
 
-static ctf_id_t
+ctf_id_t
 ctf_add_encoded (ctf_file_t *fp, uint32_t flag,
 		 const char *name, const ctf_encoding_t *ep, uint32_t kind)
 {
@@ -899,7 +905,7 @@ ctf_add_encoded (ctf_file_t *fp, uint32_t flag,
   return type;
 }
 
-static ctf_id_t
+ctf_id_t
 ctf_add_reftype (ctf_file_t *fp, uint32_t flag, ctf_id_t ref, uint32_t kind)
 {
   ctf_dtdef_t *dtd;

libctf/ctf-decls.h

@@ -72,4 +72,8 @@ void ctf_qsort_r (void *base, size_t nmemb, size_t size,
 #define MAX(a, b) ((a) > (b) ? (a) : (b))
 #define MIN(a, b) ((a) < (b) ? (a) : (b))
 
+#if !HAVE_DECL_STPCPY
+extern char *stpcpy (char *, const char *);
+#endif
+
 #endif /* _CTF_DECLS_H */

libctf/ctf-hash.c

@@ -117,6 +117,28 @@ ctf_hash_eq_type_key (const void *a, const void *b)
     && (key_a->cltk_idx == key_b->cltk_idx);
 }
 
+/* Hash a type_id_key.  */
+unsigned int
+ctf_hash_type_id_key (const void *ptr)
+{
+  ctf_helem_t *hep = (ctf_helem_t *) ptr;
+  ctf_type_id_key_t *k = (ctf_type_id_key_t *) hep->key;
+
+  return htab_hash_pointer ((void *) (uintptr_t) k->ctii_input_num)
+    + 59 * htab_hash_pointer ((void *) (uintptr_t) k->ctii_type);
+}
+
+int
+ctf_hash_eq_type_id_key (const void *a, const void *b)
+{
+  ctf_helem_t *hep_a = (ctf_helem_t *) a;
+  ctf_helem_t *hep_b = (ctf_helem_t *) b;
+  ctf_type_id_key_t *key_a = (ctf_type_id_key_t *) hep_a->key;
+  ctf_type_id_key_t *key_b = (ctf_type_id_key_t *) hep_b->key;
+
+  return (key_a->ctii_input_num == key_b->ctii_input_num)
+    && (key_a->ctii_type == key_b->ctii_type);
+}
 
 /* Hash and eq functions for the dynset.  Most of these can just use the
    underlying hashtab functions directly.   */

libctf/ctf-impl.h

@@ -247,6 +247,106 @@ typedef struct ctf_link_type_key
   ctf_id_t cltk_idx;
 } ctf_link_type_key_t;
 
+/* The structure used as the key in a cd_id_to_file_t on 32-bit platforms.  */
+typedef struct ctf_type_id_key
+{
+  int ctii_input_num;
+  ctf_id_t ctii_type;
+} ctf_type_id_key_t;
+
+/* Deduplicator state.
+
+   The dedup state below uses three terms consistently. A "hash" is a
+   ctf_dynhash_t; a "hash value" is the hash value of a type as returned by
+   ctf_dedup_hash_type; a "global type ID" or "global ID" is a packed-together
+   reference to a single ctf_file_t (by array index in an array of inputs) and
+   ctf_id_t, i.e. a single instance of some hash value in some input.
+
+   The deduplication algorithm takes a bunch of inputs and yields a single
+   shared "output" and possibly many outputs corresponding to individual inputs
+   that still contain types after sharing of unconflicted types.  Almost all
+   deduplicator state is stored in the struct ctf_dedup in the output, though a
+   (very) few things are stored in inputs for simplicity's sake, usually if they
+   are linking together things within the scope of a single TU.
+
+   Flushed at the end of every ctf_dedup run.  */
+
+typedef struct ctf_dedup
+{
+  /* The CTF linker flags in force for this dedup run.  */
+  int cd_link_flags;
+
+  /* On 32-bit platforms only, a hash of global type IDs, in the form of
+     a ctf_link_type_id_key_t.  */
+  ctf_dynhash_t *cd_id_to_file_t;
+
+  /* Atoms tables of decorated names: maps undecorated name to decorated name.
+     (The actual allocations are in the CTF file for the former and the real
+     atoms table for the latter).  Uses the same namespaces as ctf_lookups,
+     below, but has no need for null-termination.  */
+  ctf_dynhash_t *cd_decorated_names[4];
+
+  /* Map type names to a hash from type hash value -> number of times each value
+     has appeared.  */
+  ctf_dynhash_t *cd_name_counts;
+
+  /* Map global type IDs to type hash values.  Used to determine if types are
+     already hashed without having to recompute their hash values again, and to
+     link types together at later stages.  Forwards that are peeked through to
+     structs and unions are not represented in here, so lookups that might be
+     such a type (in practice, all lookups) must go via cd_replaced_types first
+     to take this into account.  Discarded before each rehashing.  */
+  ctf_dynhash_t *cd_type_hashes;
+
+  /* Maps from the names of structs/unions/enums to a a single GID which is the
+     only appearance of that type in any input: if it appears in more than one
+     input, a value which is a GID with an input_num of -1 appears.  Used in
+     share-duplicated link mode link modes to determine whether structs/unions
+     can be cited from multiple TUs.  Only populated in that link mode.  */
+  ctf_dynhash_t *cd_struct_origin;
+
+  /* Maps type hash values to a set of hash values of the types that cite them:
+     i.e., pointing backwards up the type graph.  Used for recursive conflict
+     marking.  Citations from tagged structures, unions, and forwards do not
+     appear in this graph.  */
+  ctf_dynhash_t *cd_citers;
+
+  /* Maps type hash values to input global type IDs.  The value is a set (a
+     hash) of global type IDs.  Discarded before each rehashing.  The result of
+     the ctf_dedup function.  */
+  ctf_dynhash_t *cd_output_mapping;
+
+  /* A map giving the GID of the first appearance of each type for each type
+     hash value.  */
+  ctf_dynhash_t *cd_output_first_gid;
+
+  /* Used to ensure that we never try to map a single type ID to more than one
+     hash.  */
+  ctf_dynhash_t *cd_output_mapping_guard;
+
+  /* Maps the global type IDs of structures in input TUs whose members still
+     need emission to the global type ID of the already-emitted target type
+     (which has no members yet) in the appropriate target.  Uniquely, the latter
+     ID represents a *target* ID (i.e. the cd_output_mapping of some specified
+     input): we encode the shared (parent) dict with an ID of -1.  */
+  ctf_dynhash_t *cd_emission_struct_members;
+
+  /* A set (a hash) of hash values of conflicting types.  */
+  ctf_dynset_t *cd_conflicting_types;
+
+  /* Maps type hashes to ctf_id_t's in this dictionary.  Populated only at
+     emission time, in the dictionary where emission is taking place.  */
+  ctf_dynhash_t *cd_output_emission_hashes;
+
+  /* Maps the decorated names of conflicted cross-TU forwards that were forcibly
+     emitted in this TU to their emitted ctf_id_ts.  Populated only at emission
+     time, in the dictionary where emission is taking place. */
+  ctf_dynhash_t *cd_output_emission_conflicted_forwards;
+
+  /* Points to the output counterpart of this input dictionary, at emission
+     time.  */
+  ctf_file_t *cd_output;
+} ctf_dedup_t;
 
 /* The ctf_file is the structure used to represent a CTF container to library
    clients, who see it only as an opaque pointer.  Modifications can therefore
@@ -346,6 +446,18 @@ struct ctf_file
   void *ctf_link_variable_filter_arg;           /* Argument for it. */
 
   ctf_dynhash_t *ctf_add_processing; /* Types ctf_add_type is working on now.  */
+
+  /* Atoms table for dedup string storage.  All strings in the ctf_dedup_t are
+     stored here.  Only the _alloc copy is allocated or freed: the
+     ctf_dedup_atoms may be pointed to some other CTF dict, to share its atoms.
+     We keep the atoms table outside the ctf_dedup so that atoms can be
+     preserved across multiple similar links, such as when doing cu-mapped
+     links.  */
+  ctf_dynset_t *ctf_dedup_atoms;
+  ctf_dynset_t *ctf_dedup_atoms_alloc;
+
+  ctf_dedup_t ctf_dedup;	  /* Deduplicator state.  */
+
   char *ctf_tmp_typeslice;	  /* Storage for slicing up type names.  */
   size_t ctf_tmp_typeslicelen;	  /* Size of the typeslice.  */
   void *ctf_specific;		  /* Data for ctf_get/setspecific().  */
@@ -451,11 +563,13 @@ typedef unsigned int (*ctf_hash_fun) (const void *ptr);
 extern unsigned int ctf_hash_integer (const void *ptr);
 extern unsigned int ctf_hash_string (const void *ptr);
 extern unsigned int ctf_hash_type_key (const void *ptr);
+extern unsigned int ctf_hash_type_id_key (const void *ptr);
 
 typedef int (*ctf_hash_eq_fun) (const void *, const void *);
 extern int ctf_hash_eq_integer (const void *, const void *);
 extern int ctf_hash_eq_string (const void *, const void *);
 extern int ctf_hash_eq_type_key (const void *, const void *);
+extern int ctf_hash_eq_type_id_key (const void *, const void *);
 
 extern int ctf_dynset_eq_string (const void *, const void *);
 
@@ -526,11 +640,24 @@ extern int ctf_dvd_insert (ctf_file_t *, ctf_dvdef_t *);
 extern void ctf_dvd_delete (ctf_file_t *, ctf_dvdef_t *);
 extern ctf_dvdef_t *ctf_dvd_lookup (const ctf_file_t *, const char *);
 
+extern ctf_id_t ctf_add_encoded (ctf_file_t *, uint32_t, const char *,
+				 const ctf_encoding_t *, uint32_t kind);
+extern ctf_id_t ctf_add_reftype (ctf_file_t *, uint32_t, ctf_id_t,
+				 uint32_t kind);
+
 extern void ctf_add_type_mapping (ctf_file_t *src_fp, ctf_id_t src_type,
 				  ctf_file_t *dst_fp, ctf_id_t dst_type);
 extern ctf_id_t ctf_type_mapping (ctf_file_t *src_fp, ctf_id_t src_type,
 				  ctf_file_t **dst_fp);
 
+extern int ctf_dedup_atoms_init (ctf_file_t *);
+extern int ctf_dedup (ctf_file_t *, ctf_file_t **, uint32_t ninputs,
+		      uint32_t *parents, int cu_mapped);
+extern void ctf_dedup_fini (ctf_file_t *, ctf_file_t **, uint32_t);
+extern ctf_file_t **ctf_dedup_emit (ctf_file_t *, ctf_file_t **,
+				    uint32_t ninputs, uint32_t *parents,
+				    uint32_t *noutputs, int cu_mapped);
+
 extern void ctf_decl_init (ctf_decl_t *);
 extern void ctf_decl_fini (ctf_decl_t *);
 extern void ctf_decl_push (ctf_decl_t *, ctf_file_t *, ctf_id_t);

libctf/ctf-open.c

@@ -1719,6 +1719,8 @@ ctf_file_close (ctf_file_t *fp)
   ctf_dynhash_destroy (fp->ctf_link_in_cu_mapping);
   ctf_dynhash_destroy (fp->ctf_link_out_cu_mapping);
   ctf_dynhash_destroy (fp->ctf_add_processing);
+  ctf_dedup_fini (fp, NULL, 0);
+  ctf_dynset_destroy (fp->ctf_dedup_atoms_alloc);
 
   for (err = ctf_list_next (&fp->ctf_errs_warnings); err != NULL; err = nerr)
     {