From 1d89264fc75ff0127d3b286880dedf45b19d7365 Mon Sep 17 00:00:00 2001 From: Morgan Rockett Date: Thu, 8 Aug 2024 16:29:37 -0400 Subject: [PATCH] ci: add shellcheck gh action; fix fatal shellcheck errors This commit made with the assistance of github copilot Signed-off-by: Morgan Rockett --- .github/workflows/ci.yml | 28 +++- scripts/create-e2e-report.sh | 4 +- scripts/install-build-tools.sh | 2 +- scripts/lint.sh | 9 +- scripts/native-system-benchmark.sh | 12 +- scripts/shellcheck.sh | 199 +++++++++++++++++++++++++++++ scripts/test-e2e-minikube.sh | 2 +- scripts/wait-for-it.sh | 2 +- 8 files changed, 237 insertions(+), 21 deletions(-) create mode 100755 scripts/shellcheck.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f33c274b5..869157bc1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -25,7 +25,7 @@ jobs: - uses: actions/checkout@v4 with: submodules: recursive - - name: Setup Build Env + - name: Install Build Tools run: sudo ./scripts/install-build-tools.sh - name: Setup Local Dependencies run: ./scripts/setup-dependencies.sh @@ -38,7 +38,7 @@ jobs: - uses: actions/checkout@v4 with: submodules: recursive - - name: Setup Build Env + - name: Install Build Tools run: sudo ./scripts/install-build-tools.sh - name: Setup Local Dependencies run: ./scripts/setup-dependencies.sh @@ -50,7 +50,7 @@ jobs: name: Pylint runs-on: ubuntu-22.04 continue-on-error: true - timeout-minutes: 10 + timeout-minutes: 5 strategy: matrix: python-version: ["3.10"] @@ -62,10 +62,25 @@ jobs: uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - - name: Setup Build Env + - name: Install Build Tools run: sudo ./scripts/install-build-tools.sh - name: Lint with Pylint run: ./scripts/pylint.sh + shellcheck: + name: Shellcheck + runs-on: ubuntu-22.04 + continue-on-error: true + timeout-minutes: 5 + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + - name: Install shellcheck + run: | + sudo apt-get update + sudo apt-get install -y shellcheck + - name: Lint with Shellcheck + run: ./scripts/shellcheck.sh -S error unit-and-integration-test: name: Unit and Integration Tests runs-on: ubuntu-22.04 @@ -74,7 +89,7 @@ jobs: - uses: actions/checkout@v4 with: submodules: recursive - - name: Setup Build Env + - name: Install Build Tools run: sudo ./scripts/install-build-tools.sh - name: Setup Local Dependencies run: ./scripts/setup-dependencies.sh @@ -84,7 +99,7 @@ jobs: run: ./scripts/test.sh - name: Shorten SHA id: vars - run: echo "::set-output name=sha_short::$(git rev-parse --short HEAD)" + run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_ENV - uses: actions/upload-artifact@v4 if: ${{ !env.ACT }} name: Archive Test Results @@ -114,4 +129,3 @@ jobs: name: OpenCBDC Transaction Processor docs for ${{ steps.vars.outputs.sha_short }} path: ./doxygen_generated/html/* retention-days: 7 - diff --git a/scripts/create-e2e-report.sh b/scripts/create-e2e-report.sh index 51214a4d0..771fb698c 100755 --- a/scripts/create-e2e-report.sh +++ b/scripts/create-e2e-report.sh @@ -11,9 +11,9 @@ function readAndFormatLogs() { return fi - for logfile in $(ls $logdir); do + for logfile in "$logdir"/*; do logfile_path="$logdir/$logfile" - logfile_content=$(cat $logfile_path) + logfile_content=$(<"$logfile_path") message+="\n
\n$logfile\n\n\`\`\`\n$logfile_content\n\`\`\`\n
\n" done echo "$message" diff --git a/scripts/install-build-tools.sh b/scripts/install-build-tools.sh index 523789d99..197ab904b 100755 --- a/scripts/install-build-tools.sh +++ b/scripts/install-build-tools.sh @@ -17,7 +17,7 @@ fi # Supporting these versions for buildflow PYTHON_VERSIONS=("3.10" "3.11" "3.12") -echo "Python3 versions supported: ${PYTHON_VERSIONS[@]}" +echo "Python3 versions supported: ${PYTHON_VERSIONS[*]}" # check if supported version of python3 is already installed, and save the version PY_INSTALLED='' diff --git a/scripts/lint.sh b/scripts/lint.sh index c6e4b0936..2197ab050 100755 --- a/scripts/lint.sh +++ b/scripts/lint.sh @@ -25,9 +25,12 @@ if [ -n "$whitespace_files" ] || [ -n "$newline_files" ] ; then exit 1 fi -check_format_files=$(git ls-files | grep -E "tools|tests|src|cmake-tests" \ - | grep -E "\..*pp") -clang-format --style=file --Werror --dry-run ${check_format_files[@]} +check_format_files=$(git ls-files | \ + grep -E "tools|tests|src|cmake-tests" | \ + grep -E "\..*pp") + +echo "${check_format_files}" | \ + xargs -n1 -I{} clang-format --style=file --Werror --dry-run {} if ! command -v clang-tidy &>/dev/null; then echo "clang-tidy does not appear to be installed" diff --git a/scripts/native-system-benchmark.sh b/scripts/native-system-benchmark.sh index 1b7923fa2..f295f2861 100755 --- a/scripts/native-system-benchmark.sh +++ b/scripts/native-system-benchmark.sh @@ -151,7 +151,7 @@ on_int() { printf 'Interrupting all components\n' trap '' SIGINT # avoid interrupting ourself for i in $PIDS; do # intentionally unquoted - if [[ -n "RECORD" ]]; then + if [[ -n "$RECORD" ]]; then kill -SIGINT -- "-$i" else kill -SIGINT -- "$i" @@ -194,7 +194,7 @@ on_int() { printf 'Terminating any remaining processes\n' for i in $PIDS; do # intentionally unquoted - if [[ -n "RECORD" ]]; then + if [[ -n "$RECORD" ]]; then kill -SIGTERM -- "-$i" else kill -SIGTERM -- "$i" @@ -253,7 +253,7 @@ run() { COMP= case "$RECORD" in perf) - $@ &> "$PROC_LOG" & + "$@" &> "$PROC_LOG" & COMP="$!" perf record -F 99 -a -g -o "$PNAME".perf -p "$COMP" &> "$PERF_LOG" & PERFS="$PERFS $!";; @@ -261,7 +261,7 @@ run() { ${DBG} "$@" &> "$PROC_LOG" & COMP="$!";; *) - $@ &> "$PROC_LOG" & + "$@" &> "$PROC_LOG" & COMP="$!";; esac @@ -324,7 +324,7 @@ launch() { "$RT"/scripts/wait-for-it.sh -q -t 5 -h localhost -p "$ep" done printf 'Launched logical %s %d, replica %d [PID: %d]\n' "$1" "$id" "$node" "$PID" - if [[ -n "RECORD" ]]; then + if [[ -n "$RECORD" ]]; then PIDS="$PIDS $(getpgid $PID)" else PIDS="$PIDS $PID" @@ -337,7 +337,7 @@ launch() { "$RT"/scripts/wait-for-it.sh -q -t 5 -h localhost -p "$ep" done printf 'Launched %s %d [PID: %d]\n' "$1" "$id" "$PID" - if [[ -n "RECORD" ]]; then + if [[ -n "$RECORD" ]]; then PIDS="$PIDS $(getpgid $PID)" else PIDS="$PIDS $PID" diff --git a/scripts/shellcheck.sh b/scripts/shellcheck.sh new file mode 100755 index 000000000..4318db9ae --- /dev/null +++ b/scripts/shellcheck.sh @@ -0,0 +1,199 @@ +#!/usr/bin/env bash + +RED="\e[31m" +GREEN="\e[32m" +RST_COLOR="\e[0m" + +if ! command -v shellcheck &>/dev/null; then + echo -e "${RED}[ERROR]${RST_COLOR} shellcheck is not installed." + echo "Run 'sudo ./scripts/install-build-tools.sh' to install shellcheck." + exit 1 +fi + +# Usage: ./scripts/shellcheck.sh [-e|--exclude-code=CODE] [-S|--severity=LEVEL] [-v|--view] +IFS='' read -rd '' usage <<'EOF' +Usage: %s [options] + +Options: + -h, --help print this help and exit + -e, --exclude-code exclude specific error code, can be repeated + -S, --severity=LEVEL set severity level (info, warning, error) + -v, --view view shellcheck report + + example: ./scripts/shellcheck.sh -e SC1091 -e SC1090 -S warning -v +EOF + +echo; echo "Command line arguments: $0 $*"; echo + +SEVERITY= +EXCLUDE_CODES= +VIEW="False" + +_help= +_err=0 +while [[ $# -gt 0 ]]; do + optarg= + shft_cnt=1 + if [[ "$1" = '--' ]]; then + shift 1 + break + elif [[ "$1" =~ [=] ]]; then + optarg="${1#*=}" + elif [[ "$1" =~ ^-- && $# -gt 1 && ! "$2" =~ ^- ]]; then + optarg="$2" + shft_cnt=2 + elif [[ "$1" =~ ^-[^-] && $# -gt 1 && ! "$2" =~ ^- ]]; then + optarg="$2" + shft_cnt=2 + elif [[ "$1" =~ ^-[^-] ]]; then + optarg="${1/??/}" + fi + + case "$1" in + -S*|--severity*) + # don't let the user enter -S LEVEL more than once + SEV=$(echo "${optarg}" | tr -d '[:space:]') + if [[ -n "$SEVERITY" ]]; then + printf "${RED}[Error]${RST_COLOR} Severity level already set to: %s\n" "${SEVERITY}" + _help=1; _err=1 + # valid if -S has any of 'info', 'warning', 'error' + elif [[ "${optarg}" == "info" || "${optarg}" == "warning" || "${optarg}" == "error" ]]; then + SEVERITY="$SEV" + else + # continue and disregard invalid severity level + printf "${RED}[Error]${RST_COLOR} severity level: %s\n" "${optarg}" + _err=1 + fi + shift "$shft_cnt" + ;; + -e*|--exclude-code*) + # strip whitespace from optarg + CODE=$(echo "${optarg}" | tr -d '[:space:]') + # valid if matching format SC1000-SC9999 + if [[ "${optarg}" =~ ^SC[0-9]{4}$ ]]; then + # if empty then populate with just error code, otherwise add pipe before new code for grep later + if [[ -z "${EXCLUDE_CODES}" ]]; then + EXCLUDE_CODES+="${CODE}" + else + EXCLUDE_CODES+="|${CODE}" + fi + else + # continue just don't save invalid error code + printf "${RED}[Error]${RST_COLOR} Invalid error code entered: %s\n" "${optarg}" + _err=1 + fi + shift "$shft_cnt" + ;; + -v|--view) + VIEW="True" + shift "$shft_cnt" + ;; + -h|--help) + _help=1 + ;; + *) + printf "${RED}[Error]${RST_COLOR} Unrecognized option: %s\n" "$1" + _err=1 + shift "$shft_cnt" + ;; + esac + + # exit on help message + if [[ $_help -eq 1 ]]; then + printf "%s %s\n" "$usage" "$(basename "$0")" + exit 0 + fi + # continue on invalid arg, let user know but don't exit + if [[ "$_err" -eq 1 ]]; then + printf "${RED}[Error]${RST_COLOR} Invalid argument: %s\n" "${optarg}" + printf "%s %s\n" "$usage" "$(basename "$0")" + fi +done + +# if severity not set, set it to error as default +if [[ -z "$SEVERITY" ]]; then + SEVERITY="error" +fi + +ROOT="$(cd "$(dirname "$0")"/.. && pwd)" +SHELLCHECK_REPORT="${ROOT}/shellcheck-report.txt" + +NUM_CORES=1 +if [[ "$OSTYPE" == "linux-gnu"* ]]; then + NUM_CORES=$(grep -c ^processor /proc/cpuinfo) +elif [[ "$OSTYPE" == "darwin"* ]]; then + NUM_CORES=$(sysctl -n hw.ncpu) +fi + +# run shellcheck in parallel on all tracked shell scripts +# +# checking status of this run will give failure if info/warning/error is found by default +# +# determine status by parsing shellcheck report to see if any messages +# of the severity level or more strict are present to determine failure (true errors) + +# check if git is installed +if command -v git &>/dev/null; then + echo "Using git ls-files to find shell scripts..."; echo + git ls-files '*.sh' | xargs -n 1 -P "$NUM_CORES" shellcheck > "$SHELLCHECK_REPORT" +else + echo "git is not installed. Using find to compile list of shell scripts..."; echo + if [[ -z "$EXCLUDE_CODES" ]]; then + find "$ROOT" -name '*.sh' -print0 | xargs -0 -n 1 -P "$NUM_CORES" shellcheck > "$SHELLCHECK_REPORT" + fi +fi + +# if shell check report exists to determine if shellcheck run was successful +if [[ -z "$SHELLCHECK_REPORT" ]]; then + echo "${RED}[FAIL]${RST_COLOR}Shellcheck report ${SHELLCHECK_REPORT} not found. Exiting..." + exit 1 +else + if [[ ! -s "$SHELLCHECK_REPORT" ]]; then + echo "Shellcheck report is empty: ${SHELLCHECK_REPORT}" + echo "Either there are no info/warning/error messages for all shell scripts" + echo "in the codebase or shellcheck failed to run successfully. Exiting..." + exit 0 + fi +fi + +# view non-empty shellcheck report, includes info, warnings, errors +if [[ "$VIEW" == "True" ]]; then + echo "Shellcheck report: ${SHELLCHECK_REPORT}" + cat "$SHELLCHECK_REPORT" +fi + +# detect if fatal errors are in shellcheck report +echo "Checking for errors in shellcheck report with severity level ${SEVERITY}" +# print exclude codes and replace pipe with ', ' +READABLE_EXCLUDE_CODES=("${EXCLUDE_CODES//|/, }") +echo "Excluding error codes: ${READABLE_EXCLUDE_CODES[*]}"; echo + +# if any messages of severity level or more strict are present, use for grepping report +case "$SEVERITY" in + "info") REGEX_SEVERITY="info|warning|error" ;; + "warning") REGEX_SEVERITY="warning|error" ;; + *) REGEX_SEVERITY="error" ;; +esac + +# just grep report for severity level if no exclude codes, otherwise pipe and exclude codes from matches +SEARCH_CMD() { + if [[ "${#EXCLUDE_CODES[@]}" -eq 0 ]]; then + grep -E "\(${REGEX_SEVERITY}\):" "$SHELLCHECK_REPORT" + else + grep -E "\(${REGEX_SEVERITY}\):" "$SHELLCHECK_REPORT" | grep -v -E "${EXCLUDE_CODES}" + fi +} + +# if grep yielded no output then no violations of severity level or higher found (success) +SEARCH_RESULTS="$(SEARCH_CMD)" +if [[ -z "$SEARCH_RESULTS" ]]; then + echo -e "${GREEN}[PASS]${RST_COLOR} Shellcheck did not detect violations scanning with severity level '${SEVERITY}'" + echo; echo -e "${GREEN}Shellcheck passed.${RST_COLOR} See report: ${SHELLCHECK_REPORT}"; echo + exit 0 +else + COUNT=$(echo "$SEARCH_RESULTS" | wc -l | tr -d '[:space:]') + echo -e "${RED}[FAIL]${RST_COLOR} Shellcheck found ${RED}${COUNT}${RST_COLOR}"\ + "unexcused violations scanning with severity level '${SEVERITY}'"; echo + echo -e "${RED}Shellcheck failed.${RST_COLOR} See report: ${SHELLCHECK_REPORT}"; echo + exit 1 +fi diff --git a/scripts/test-e2e-minikube.sh b/scripts/test-e2e-minikube.sh index d25900163..b4dc55759 100755 --- a/scripts/test-e2e-minikube.sh +++ b/scripts/test-e2e-minikube.sh @@ -7,7 +7,7 @@ BUILD_DOCKER=${TESTRUN_BUILD_DOCKER:-1} # Make sure we have the necessary tools installed required_executables=(minikube docker go helm kubectl) -for e in ${required_executables[@]}; do +for e in "${required_executables[@]}"; do if ! command -v $e &> /dev/null; then echo "'$e' command not be found! This is required to run. Please install it." exit 1 diff --git a/scripts/wait-for-it.sh b/scripts/wait-for-it.sh index d990e0d36..df6ad2646 100755 --- a/scripts/wait-for-it.sh +++ b/scripts/wait-for-it.sh @@ -150,7 +150,7 @@ if [[ $WAITFORIT_TIMEOUT_PATH =~ "busybox" ]]; then WAITFORIT_ISBUSY=1 # Check if busybox timeout uses -t flag # (recent Alpine versions don't support -t anymore) - if timeout &>/dev/stdout | grep -q -e '-t '; then + if timeout |& tee /dev/stdout | grep -q -e '-t '; then WAITFORIT_BUSYTIMEFLAG="-t" fi else