Skip to content

Commit

Permalink
Provide gpdemo deployment script
Browse files Browse the repository at this point in the history
Customers have complained that the deployment procedures of CBDB is
tedious. In this commit, gpdemo provides a default configuration to enable
a one-click deployment experience. The default parameters are the same
as "make create-demo-cluster".
  • Loading branch information
Ray-Eldath committed Nov 15, 2023
1 parent 5a6cac3 commit e221d14
Show file tree
Hide file tree
Showing 7 changed files with 154 additions and 66 deletions.
36 changes: 6 additions & 30 deletions gpAux/gpdemo/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -9,31 +9,7 @@
top_builddir = ../..
-include $(top_builddir)/src/Makefile.global

#
# VARIABLES
#

export enable_gpfdist
export with_openssl

PORT_BASE ?= 7000
NUM_PRIMARY_MIRROR_PAIRS ?= 3
BLDWRAP_POSTGRES_CONF_ADDONS := fsync=off $(BLDWRAP_POSTGRES_CONF_ADDONS)

ifeq ($(WITH_MIRRORS), )
WITH_MIRRORS = true
endif

ifeq ($(WITH_MIRRORS), true)
WITH_STANDBY = true
endif

export DEMO_PORT_BASE=$(PORT_BASE)
export NUM_PRIMARY_MIRROR_PAIRS
export WITH_MIRRORS
export WITH_STANDBY
export BLDWRAP_POSTGRES_CONF_ADDONS
export DEFAULT_QD_MAX_CONNECT=150
SHELL := /usr/bin/bash

all:
$(MAKE) clean
Expand All @@ -42,22 +18,22 @@ all:
$(MAKE) probe

cluster create-demo-cluster:
@./demo_cluster.sh
@source gpdemo-defaults.sh && ./demo_cluster.sh
@echo ""

probe:
@./probe_config.sh
@source gpdemo-defaults.sh && ./probe_config.sh
@echo ""

check:
@./demo_cluster.sh -c
check:
@source gpdemo-defaults.sh && ./demo_cluster.sh -c
@echo ""

clean destroy-demo-cluster:
@echo "======================================================================"
@echo "Deleting cluster.... "
@echo "======================================================================"
@./demo_cluster.sh -d
@source gpdemo-defaults.sh && ./demo_cluster.sh -d
@echo ""

.PHONY: all cluster create-demo-cluster create-segwalrep-cluster probe clean check clean destroy-demo-cluster
32 changes: 18 additions & 14 deletions gpAux/gpdemo/README
Original file line number Diff line number Diff line change
Expand Up @@ -31,11 +31,7 @@ RUNNING GP DEMO

su - gpadmin

2. Go into the gpdemo directory:

cd gpAux/gpdemo

3. Source greenplum_path.sh
2. Source greenplum_path.sh

. /usr/local/greenplum-db/greenplum_path.sh

Expand All @@ -44,33 +40,38 @@ RUNNING GP DEMO
The "." will source the greenplum_path.sh into the current
shell, instead of starting a subprocess.

4. Create the cluster:
3. Create the cluster at the current directory:

gpdemo

make cluster
By default, all data of the demo cluster will be stored in
"./datadirs" of the current directory. You may change this by
cd to another directory or using "DATADIRS" environment variable
described in the next section.

5. Source gpAux/gpdemo/gpdemo-env.sh
4. Source ./gpdemo-env.sh

6. If setup completes correctly, you will have a virtual Cloudberry
5. If setup completes correctly, you will have a virtual Cloudberry
Database system running on the local host. You can connect to it
using the psql client program as follows:

psql postgres

7. To exit the psql client program:
6. To exit the psql client program:

\q

8. See the Cloudberry Database User Guide for more information on
7. See the Cloudberry Database User Guide for more information on
using your Cloudberry Database system.


******************************************************************
REMOVING GP DEMO
******************************************************************

1. From the gpdemo directory run make clean:
1. From the gpdemo directory run "gpdemo -c":

make clean
gpdemo -d

2. This will stop all Cloudberry Database demo server processes
and remove all files and directories created by the GP demo
Expand All @@ -96,7 +97,10 @@ environment variables before initializing the cluster:
export NUM_PRIMARY_MIRROR_PAIRS=<number segments>

It is recommended to specify an odd number of segments, in order
to catch problems with data distribution.
to catch problems with data distribution. When set to 0, a singlenode
deployment will be created. Under this mode, there are no segments,
only one coordinator (QD) and one coordinator standby. All data sit
on the two nodes and nothing gets distributed.

* Data directory:

Expand Down
26 changes: 9 additions & 17 deletions gpAux/gpdemo/demo_cluster.sh
Original file line number Diff line number Diff line change
Expand Up @@ -102,16 +102,6 @@ checkDemoConfig(){
return 0
}

USAGE(){
echo ""
echo " `basename $0` {-c | -d | -u} <-K>"
echo " -c : Check if demo is possible."
echo " -d : Delete the demo."
echo " -K : Create cluster without data checksums."
echo " -u : Usage, prints this message."
echo ""
}

#
# Clean up the demo
#
Expand Down Expand Up @@ -140,6 +130,10 @@ cleanDemo(){
echo "Deleting clusterConfigFile"
rm -f clusterConfigFile
fi
if [ -f clusterConfigPostgresAddonsFile ]; then
echo "Deleting clusterConfigPostgresAddonsFile"
rm -f clusterConfigPostgresAddonsFile
fi
if [ -d ${DATADIRS} ]; then
echo "Deleting ${DATADIRS}"
rm -rf ${DATADIRS}
Expand All @@ -155,10 +149,9 @@ cleanDemo(){
# Main Section
#*****************************************************************************

while getopts ":cdK'?'" opt
while getopts ":cdK" opt
do
case $opt in
'?' ) USAGE ;;
case $opt in
c) checkDemoConfig
RETVAL=$?
if [ $RETVAL -ne 0 ]; then
Expand All @@ -173,9 +166,7 @@ do
K) DATACHECKSUMS=0
shift
;;
*) USAGE
exit 0
;;
*) USAGE; exit 1;;
esac
done

Expand Down Expand Up @@ -302,7 +293,7 @@ cat >> $CLUSTER_CONFIG <<-EOF
COORDINATOR_PORT=${COORDINATOR_DEMO_PORT}
# Shell to use to execute commands on all hosts
TRUSTED_SHELL="`pwd`/lalshell"
TRUSTED_SHELL="$(dirname "$0")/lalshell"
ENCODING=UNICODE
EOF
Expand Down Expand Up @@ -487,6 +478,7 @@ cat > gpdemo-env.sh <<-EOF
export PGPORT=${COORDINATOR_DEMO_PORT}
export COORDINATOR_DATA_DIRECTORY=$QDDIR/${SEG_PREFIX}-1
export MASTER_DATA_DIRECTORY=$QDDIR/${SEG_PREFIX}-1
export NUM_PRIMARY_MIRROR_PAIRS=${NUM_PRIMARY_MIRROR_PAIRS}
EOF

if [ "${RETURN}" -gt 1 ];
Expand Down
38 changes: 38 additions & 0 deletions gpAux/gpdemo/gpdemo-defaults.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
#!/usr/bin/env bash

USAGE() {
echo ""
echo " `basename $0` {-c | -d | -h} <-K>"
echo " -c : Check if demo is possible."
echo " -d : Delete the demo."
echo " -K : Create cluster without data checksums."
echo " -h : Usage, prints this message."
echo ""
}

PORT_BASE=${PORT_BASE:=7000}
NUM_PRIMARY_MIRROR_PAIRS=${NUM_PRIMARY_MIRROR_PAIRS:=3}
if [ "${NUM_PRIMARY_MIRROR_PAIRS}" -eq "0" ]; then
BLDWRAP_POSTGRES_CONF_ADDONS="fsync=on ${BLDWRAP_POSTGRES_CONF_ADDONS}" # always enable fsync if singlenode
fi
if [[ ! ${BLDWRAP_POSTGRES_CONF_ADDONS} == *"fsync=on"* ]]; then
BLDWRAP_POSTGRES_CONF_ADDONS="fsync=off ${BLDWRAP_POSTGRES_CONF_ADDONS}" # only append fsync=off if it doesn't contains fsync=on
else
BLDWRAP_POSTGRES_CONF_ADDONS="${BLDWRAP_POSTGRES_CONF_ADDONS}"
fi
WITH_MIRRORS=${WITH_MIRRORS:="true"}
if [[ "${WITH_MIRRORS}" == "true" ]]; then
WITH_STANDBY="true"
fi

export -f USAGE

export enable_gpfdist
export with_openssl

export DEMO_PORT_BASE="$PORT_BASE"
export NUM_PRIMARY_MIRROR_PAIRS
export WITH_MIRRORS
export WITH_STANDBY
export BLDWRAP_POSTGRES_CONF_ADDONS
export DEFAULT_QD_MAX_CONNECT=150
8 changes: 4 additions & 4 deletions gpAux/gpdemo/probe_config.sh
Original file line number Diff line number Diff line change
Expand Up @@ -44,13 +44,13 @@ declare -a PORTS=(5432 10001 10002 10003)
((PORT_MIN=0))
((PORT_MAX=$NUM_PRIMARY_MIRROR_PAIRS))

if [ -z $DEMO_PORT_BASE ] ; then
echo "set DEMO_PORT_BASE"
if [ -z $PORT_BASE ] ; then
echo "set PORT_BASE"
exit 1
fi

declare -a PORTS=(`expr $DEMO_PORT_BASE` \
`expr $DEMO_PORT_BASE + 2` `expr $DEMO_PORT_BASE + 3 ` `expr $DEMO_PORT_BASE + 4`)
declare -a PORTS=(`expr $PORT_BASE` \
`expr $PORT_BASE + 2` `expr $PORT_BASE + 3 ` `expr $PORT_BASE + 4`)

#
# Check tables on Coordinator
Expand Down
13 changes: 12 additions & 1 deletion gpMgmt/bin/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -16,16 +16,24 @@ PROGRAMS= analyzedb gpactivatestandby gpaddmirrors gpcheckcat gpcheckperf \
gpcheckresgroupimpl gpconfig gpdeletesystem gpexpand gpinitstandby \
gpinitsystem gpload gpload.py gplogfilter gpmovemirrors \
gppkg gprecoverseg gpreload gpscp gpsd gpssh gpssh-exkeys gpstart \
gpstate gpstop minirepro gpmemwatcher gpmemreport
gpstate gpstop minirepro gpmemwatcher gpmemreport gpdemo

GPDEMO_LIBS = gpdemo-defaults.sh lalshell generate_certs.sh demo_cluster.sh \
probe_config.sh README

installdirs:
$(MKDIR_P) '$(DESTDIR)$(bindir)/lib'
$(MKDIR_P) '$(DESTDIR)$(bindir)/lib/gpdemo'

installprograms: installdirs
for file in $(PROGRAMS); do \
$(INSTALL_SCRIPT) $$file '$(DESTDIR)$(bindir)/'$$file ; \
$(PERL) $(top_builddir)/putversion '$(DESTDIR)$(bindir)/'$$file ; \
done
# install dependencies of gpdemo
for file in $(GPDEMO_LIBS); do \
$(INSTALL_SCRIPT) $(top_builddir)/gpAux/gpdemo/$$file '$(DESTDIR)$(bindir)/lib/gpdemo/'$$file ; \
done
# Symlink gpcheckcat from bin to bin/lib to maintain backward compatibility
if [ ! -L $(DESTDIR)$(bindir)/lib/gpcheckcat ]; then \
cd $(DESTDIR)$(bindir)/lib/ && $(LN_S) ../gpcheckcat gpcheckcat; \
Expand All @@ -36,6 +44,9 @@ uninstall:
for file in $(PROGRAMS); do \
rm -f '$(DESTDIR)$(bindir)/'$$file ; \
done
for file in $(GPDEMO_LIBS); do \
rm -f '$(DESTDIR)$(bindir)/lib/gpdemo/'$$file ; \
done
rm -f '$(DESTDIR)$(bindir)/gpload.bat'

#
Expand Down
67 changes: 67 additions & 0 deletions gpMgmt/bin/gpdemo
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
#!/usr/bin/env bash

WORKDIR=$(dirname "$0")
GPDEMO_LIB=$WORKDIR/lib/gpdemo

README() {
cat "$GPDEMO_LIB"/README
}

PROBE_CONFIG() {
source_gpdemo_defaults
"$GPDEMO_LIB"/probe_config.sh
}

DEMO_CLUSTER() {
source_gpdemo_defaults
"$GPDEMO_LIB"/demo_cluster.sh "$@"
}

source_gpdemo_defaults() {
. "$GPDEMO_LIB"/gpdemo-defaults.sh

# overwrite the shorter usage defined in gpdemo-defaults.sh
USAGE() {
echo ""
echo " gpdemo {-c | -d | -p | -h | -H | -v} <-K>"
echo " : Default behaviour (no argument) is to create demo cluster."
echo " -c : Check if demo is possible."
echo " -d : Delete the demo."
echo " -K : Create cluster without data checksums."
echo " -p : Probe configuration of an alive cluster."
echo " -h : Usage, prints this message."
echo " -H : Detailed usage."
echo " -v : Show version."
echo ""
echo " There are few environment variables (e.g., PORT_BASE, DATADIRS) to"
echo " further config the demo cluster. See \"gpdemo -H\" for more info."
echo ""
}
}

CHECK_FILENAME=("demo_cluster.sh" "probe_config.sh" "gpdemo-defaults.sh" "lalshell" "README" "../gp_bash_version.sh")
for filename in "${CHECK_FILENAME[@]}"; do
checking="$GPDEMO_LIB"/"$filename"
if [[ ! -f "$checking" ]]; then
echo "Missing $checking"
echo ""
echo "This may be caused by incomplete installation or packaging error."
echo "Try to re-install CloudBerryDB. If it doesn't help, contact support."
exit 1
fi
done

. "$WORKDIR"/lib/gp_bash_version.sh

while getopts ":pHv" opt
do
case $opt in
p) PROBE_CONFIG; exit 0;;
H) README; exit 0;;
v) print_version; exit 0;;
*) DEMO_CLUSTER "$@"; exit $?;; # fallback to demo_cluster.sh
esac
done

# default behaviour is fallback to "demo_cluster.sh" (no argument)
DEMO_CLUSTER

0 comments on commit e221d14

Please sign in to comment.