Feature: building postgres backend into shared library

Building postgres backend into shared library allows multiple postmaster
processes to share postgres.so, thereby reducing the memory overhead of
a single machine with multiple segments. We have added a
--enable-shared-postgres-backend compilation parameter to compile and
generate postgres.so. We only test it locally, please use it with caution.

configure

@@ -758,9 +758,10 @@ enable_ic_proxy
 enable_external_fts
 HAVE_CXX14
 enable_gpcloud
+enable_shared_postgres_backend
 enable_mapreduce
-enable_orca
 enable_catalog_ext
+enable_orca
 autodepend
 PKG_CONFIG_LIBDIR
 PKG_CONFIG_PATH
@@ -895,6 +896,7 @@ enable_cassert
 enable_orca
 enable_catalog_ext
 enable_mapreduce
+enable_shared_postgres_backend
 enable_gpcloud
 enable_external_fts
 enable_ic_proxy
@@ -1603,8 +1605,10 @@ Optional Features:
   --enable-depend         turn on automatic dependency tracking
   --enable-cassert        enable assertion checks (for debugging)
   --disable-orca          disable ORCA optimizer
-  --enable-catalog-ext    enable CloudberryDB catalog extension
-  --enable-mapreduce      enable CloudberryDB Mapreduce support
+  --enable-catalog-ext    enable Cloudberry catalog extension
+  --enable-mapreduce      enable Cloudberry Mapreduce support
+  --enable-shared-postgres-backend
+                          enable Cloudberry shared postgres backend support
   --enable-gpcloud        enable gpcloud support
   --enable-external-fts   enable external fts support
   --enable-ic-proxy       enable interconnect proxy mode (requires libuv
@@ -1648,7 +1652,7 @@ Optional Packages:
   --with-ldap             build with LDAP support
   --with-bonjour          build with Bonjour support
   --with-selinux          build with SELinux support
-  --with-libseccomp       build with libseccomp support
+  --with-libseccomp       build with Libseccomp support
   --with-systemd          build with systemd support
   --without-readline      do not use GNU Readline nor BSD Libedit for editing
   --with-libedit-preferred
@@ -8309,9 +8313,8 @@ $as_echo "#define GPOS_Darwin 1" >>confdefs.h
 
 fi
 
-
 #
-# Enable catalog extension
+# --enable-catalog-ext enables catalog extension
 #
 
 
@@ -8338,8 +8341,9 @@ else
 fi
 
 
-{ $as_echo "$as_me:${as_lineno-$LINENO}: result: checking whether to build with CloudberryDB catalog extension... $enable_catalog_ext" >&5
-$as_echo "checking whether to build with CloudberryDB catalog extension ... $enable_catalog_ext" >&6; }
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: checking whether to build with catalog extension... $enable_catalog_ext" >&5
+$as_echo "checking whether to build with catalog extension... $enable_catalog_ext" >&6; }
+
 
 #
 # --enable-mapreduce enables GPMapreduce support
@@ -8369,10 +8373,49 @@ else
 fi
 
 
-{ $as_echo "$as_me:${as_lineno-$LINENO}: result: checking whether to build with CloudberryDB Mapreduce... $enable_mapreduce" >&5
-$as_echo "checking whether to build with CloudberryDB Mapreduce... $enable_mapreduce" >&6; }
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: checking whether to build with Cloudberry Mapreduce... $enable_mapreduce" >&5
+$as_echo "checking whether to build with Cloudberry Mapreduce... $enable_mapreduce" >&6; }
 
 
+#
+# --enable-shared-postgres-backend enables shared postgres backend support
+#
+
+
+# Check whether --enable-shared-postgres-backend was given.
+if test "${enable_shared_postgres_backend+set}" = set; then :
+  enableval=$enable_shared_postgres_backend;
+  case $enableval in
+    yes)
+
+$as_echo "#define USE_SHARED_POSTGRES_BACKEND 1" >>confdefs.h
+
+      ;;
+    no)
+      :
+      ;;
+    *)
+      as_fn_error $? "no argument expected for --enable-shared-postgres-backend option" "$LINENO" 5
+      ;;
+  esac
+
+else
+  enable_shared_postgres_backend=no
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: checking whether to build with Cloudberry shared postgres backend... $enable_shared_postgres_backend" >&5
+$as_echo "checking whether to build with Cloudberry shared postgres backend... $enable_shared_postgres_backend" >&6; }
+
+
+if test "$enable_shared_postgres_backend" = yes ; then :
+   # then
+  CFLAGS="$CFLAGS -fPIC "
+  CXXFLAGS="$CXXFLAGS -fPIC "
+
+fi # fi
+
 #
 # gpcloud, enabled by default
 #
@@ -9600,10 +9643,10 @@ fi
 $as_echo "$with_selinux" >&6; }
 
 #
-# libseccomp
+# Libseccomp
 #
-{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with libseccomp support" >&5
-$as_echo_n "checking whether to build with libseccomp support... " >&6; }
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to build with Libseccomp support" >&5
+$as_echo_n "checking whether to build with Libseccomp support... " >&6; }
 
 
 
@@ -9627,6 +9670,8 @@ else
 
 fi
 
+
+
 { $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_libseccomp" >&5
 $as_echo "$with_libseccomp" >&6; }
 
@@ -11717,10 +11762,10 @@ else
             break 2
           else
             { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING:
-*** The installed version of Flex, $pgac_candidate, is too old to use with CloudberryDB.
+*** The installed version of Flex, $pgac_candidate, is too old to use with Cloudberry DB.
 *** Flex version 2.5.4 or later is required, but this is $pgac_flex_version." >&5
 $as_echo "$as_me: WARNING:
-*** The installed version of Flex, $pgac_candidate, is too old to use with CloudberryDB.
+*** The installed version of Flex, $pgac_candidate, is too old to use with Cloudberry DB.
 *** Flex version 2.5.4 or later is required, but this is $pgac_flex_version." >&2;}
           fi
         fi
@@ -15607,7 +15652,7 @@ fi
 
 fi
 
-if test "$with_libseccomp" = yes ; then
+if test "$with_libseccomp" = yes; then
   { $as_echo "$as_me:${as_lineno-$LINENO}: checking for seccomp_init in -lseccomp" >&5
 $as_echo_n "checking for seccomp_init in -lseccomp... " >&6; }
 if ${ac_cv_lib_seccomp_seccomp_init+:} false; then :
@@ -16821,7 +16866,7 @@ fi
 
 if test "$enable_mapreduce" = yes; then
   if test "$with_perl" = no; then
-    as_fn_error $? "CloudberryDB Mapreduce requires Perl, reconfigure with --with-perl" "$LINENO" 5
+    as_fn_error $? "Cloudberry Mapreduce requires Perl, reconfigure with --with-perl" "$LINENO" 5
   fi
   for ac_header in yaml.h
 do :
@@ -16832,7 +16877,7 @@ if test "x$ac_cv_header_yaml_h" = xyes; then :
 _ACEOF
 
 else
-  as_fn_error $? "header file <yaml.h> is required for CloudberryDB Mapreduce" "$LINENO" 5
+  as_fn_error $? "header file <yaml.h> is required for Cloudberry Mapreduce" "$LINENO" 5
 fi
 
 done
@@ -18435,7 +18480,7 @@ _ACEOF
 
 else
 
-	as_fn_error $? "getifaddrs and inet_ntop are required for CloudberryDB" "$LINENO" 5
+	as_fn_error $? "getifaddrs and inet_ntop are required for Cloudberry" "$LINENO" 5
 
 fi
 done
@@ -22774,7 +22819,7 @@ fi
 BLD_ARCH=`echo $BLD_ARCH`
 
 
-# Get the CloudberryDB version string from VERSION
+# Get the Cloudberry version string from VERSION
 
 GP_VERSION_LONG=`bash ./getversion`
 

configure.ac

@@ -855,6 +855,21 @@ PGAC_ARG_BOOL(enable, mapreduce, no, [enable Cloudberry Mapreduce support],
 AC_MSG_RESULT([checking whether to build with Cloudberry Mapreduce... $enable_mapreduce])
 AC_SUBST(enable_mapreduce)
 
+#
+# --enable-shared-postgres-backend enables shared postgres backend support
+#
+PGAC_ARG_BOOL(enable, shared-postgres-backend, no, [enable Cloudberry shared postgres backend support],
+              [AC_DEFINE([USE_SHARED_POSTGRES_BACKEND], 1,
+			             [Define to 1 to build with shared postgres backend capabilities (--enable-shared-postgres-backend)])])
+AC_MSG_RESULT([checking whether to build with Cloudberry shared postgres backend... $enable_shared_postgres_backend])
+AC_SUBST(enable_shared_postgres_backend)
+
+AS_IF([test "$enable_shared_postgres_backend" = yes ],
+[ # then
+  CFLAGS="$CFLAGS -fPIC "
+  CXXFLAGS="$CXXFLAGS -fPIC "
+]) # fi
+
 #
 # gpcloud, enabled by default
 #
@@ -1072,6 +1087,14 @@ PGAC_ARG_BOOL(with, selinux, no, [build with SELinux support])
 AC_SUBST(with_selinux)
 AC_MSG_RESULT([$with_selinux])
 
+#
+# Libseccomp
+#
+AC_MSG_CHECKING([whether to build with Libseccomp support])
+PGAC_ARG_BOOL(with, libseccomp, no, [build with Libseccomp support])
+AC_SUBST(with_libseccomp)
+AC_MSG_RESULT([$with_libseccomp])
+
 #
 # Systemd
 #
@@ -1712,6 +1735,11 @@ if test "$with_selinux" = yes; then
                [AC_MSG_ERROR([library 'libselinux', version 2.1.10 or newer, is required for SELinux support])])
 fi
 
+if test "$with_libseccomp" = yes; then
+  AC_CHECK_LIB(seccomp, seccomp_init, [],
+               [AC_MSG_ERROR([library 'libseccomp' is required for Seccomp BPF support])])
+fi
+
 # for contrib/uuid-ossp
 if test "$with_uuid" = bsd ; then
   # On BSD, the UUID functions are in libc

src/Makefile.global.in

@@ -228,6 +228,7 @@ enable_pxf			= @enable_pxf@
 enable_debug_extensions = @enable_debug_extensions@
 enable_orafce		= @enable_orafce@
 enable_mapreduce    = @enable_mapreduce@
+enable_shared_postgres_backend    = @enable_shared_postgres_backend@
 enable_gpcloud 	    = @enable_gpcloud@
 enable_ic_proxy     = @enable_ic_proxy@
 

src/backend/Makefile

@@ -47,12 +47,18 @@ LOCALOBJS += utils/probes.o
 endif
 endif
 
+# FIXME: The --enable-shared-postgres-backend parameter build the postgres.so shared between postmaster
+# But we only test it locally without CI test. Please use it with caution.
+ifeq ($(enable_shared_postgres_backend),yes)
+OBJS = $(LOCALOBJS) $(SUBDIROBJS)
+else
 OBJS = \
 	$(LOCALOBJS) \
 	$(SUBDIROBJS) \
 	$(top_builddir)/src/common/libpgcommon_srv.a \
 	$(top_builddir)/src/port/libpgport_srv.a
 
+endif
 ifeq ($(PORTNAME), darwin)
 LDOPTS = -Z
 endif
@@ -79,8 +85,18 @@ ifneq ($(PORTNAME), cygwin)
 ifneq ($(PORTNAME), win32)
 ifneq ($(PORTNAME), aix)
 
+ifeq ($(enable_shared_postgres_backend),yes)
+postgres.so: $(OBJS)
+	$(CXX) -shared $(CXXFLAGS) $(LDFLAGS) $(LDFLAGS_EX) $(export_dynamic) $(filter-out main/main.o, $(call expand_subsys,$^)) -o $@
+
+postgres: main/main.o postgres.so $(top_builddir)/src/port/libpgport_srv.a $(top_builddir)/src/common/libpgcommon_srv.a
+	$(CXX) $(CXXFLAGS) $(LDFLAGS) $(LDFLAGS_EX) $(export_dynamic) \
+	  main/main.o postgres.so $(top_builddir)/src/port/libpgport_srv.a \
+	  $(top_builddir)/src/common/libpgcommon_srv.a $(LIBS) -o $@
+else
 postgres: $(OBJS)
 	$(CXX) $(CXXFLAGS) $(call expand_subsys,$^) $(LDFLAGS) $(LDFLAGS_EX) $(export_dynamic) $(LIBS) -o $@
+endif
 
 endif
 endif
@@ -136,9 +152,13 @@ $(top_builddir)/src/port/libpgport_srv.a: | submake-libpgport
 
 # The postgres.o target is needed by the rule in Makefile.global that
 # creates the exports file when MAKE_EXPORTS = true.
+ifeq ($(enable_shared_postgres_backend),yes)
+postgres.o: $(OBJS)  $(top_builddir)/src/port/libpgport_srv.a $(top_builddir)/src/common/libpgcommon_srv.a
+	$(CXX) $(LDREL) $(call expand_subsys,$^) $(LDFLAGS) $(LDFLAGS_EX) $(LIBS) -o $@
+else
 postgres.o: $(OBJS)
 	$(CXX) $(LDREL) $(call expand_subsys,$^) $(LDFLAGS) $(LDFLAGS_EX) $(LIBS) -o $@
-
+endif
 
 # The following targets are specified in make commands that appear in
 # the make files in our subdirectories. Note that it's important we
@@ -251,6 +271,9 @@ endif
 
 install-bin: postgres $(POSTGRES_IMP) installdirs
 	$(INSTALL_PROGRAM) postgres$(X) '$(DESTDIR)$(bindir)/postgres$(X)'
+ifeq ($(enable_shared_postgres_backend),yes)
+	$(INSTALL_PROGRAM) postgres.so '$(DESTDIR)$(libdir)/postgres.so'
+endif
 ifneq ($(PORTNAME), win32)
 	@rm -f $(DESTDIR)$(bindir)/postmaster$(X)
 	ln -s postgres$(X) $(DESTDIR)$(bindir)/postmaster$(X)
@@ -287,6 +310,9 @@ endif
 
 uninstall:
 	rm -f $(DESTDIR)$(bindir)/postgres$(X) $(DESTDIR)$(bindir)/postmaster $(DESTDIR)$(bindir)/cdbsyncmaster
+ifeq ($(enable_shared_postgres_backend),yes)
+	rm -f $(DESTDIR)$(bindir)/postgres$(X) $(DESTDIR)$(libdir)/postgres.so $(DESTDIR)$(bindir)/postmaster $(DESTDIR)$(bindir)/cdbsyncmaster
+endif
 ifeq ($(MAKE_EXPORTS), true)
 	rm -f '$(DESTDIR)$(pkglibdir)/$(POSTGRES_IMP)'
 	rm -f '$(DESTDIR)$(pgxsdir)/$(MKLDEXPORT_DIR)/mkldexport.sh'
@@ -322,6 +348,9 @@ endif
 
 clean:
 	rm -f $(LOCALOBJS) postgres$(X) $(POSTGRES_IMP)
+ifeq ($(enable_shared_postgres_backend),yes)
+	rm -f $(LOCALOBJS) postgres.so $(POSTGRES_IMP)
+endif
 ifeq ($(PORTNAME), cygwin)
 	rm -f postgres.dll libpostgres.a
 endif

src/include/pg_config.h.in

@@ -1042,6 +1042,9 @@
 /* Define to 1 to build with Mapreduce capabilities (--enable-mapreduce) */
 #undef USE_MAPREDUCE
 
+/* Define to 1 to build with shared postgres backend capabilities (--enable-shared-postgres-backend) */
+#undef USE_SHARED_POSTGRES_BACKEND
+
 /* Define to 1 to build with LZ4 support. (--with-lz4) */
 #undef USE_LZ4