Merge pull request #3012 from MoralCode/backend-docker-builder

[backend] refactor docker container to builder pattern

docker/backend/Dockerfile

@@ -1,4 +1,34 @@
 # SPDX-License-Identifier: MIT
+
+
+FROM golang:1.23 AS golang
+
+
+# RUN ./scripts/docker/install-go.sh
+ENV PATH="${PATH}:/usr/local/go/bin"
+COPY ./scripts/ scripts/
+RUN ./scripts/docker/install-golang-deps.sh
+
+
+# FROM rust:1.78 as rust
+
+# # Ensure Rust directories are writable
+# RUN mkdir -p /root/.rustup/downloads /root/.cargo/registry && \
+#     chmod -R 777 /root/.rustup /root/.cargo
+
+# # Add rust and cargo to PATH
+# # ENV PATH="/root/.cargo/bin:${PATH}"
+
+# # Install the specific version of Rust
+# # RUN set -x \
+# #     && rustup install 1.78.0
+# # RUN set -x \
+# #     && rustup default 1.78.0
+
+
+# # Add rust and cargo to PATH
+# ENV PATH="/usr/bin/:/root/.cargo/bin:/usr/local/bin:${PATH}"
+
 FROM python:3.11-slim-bullseye
 
 LABEL maintainer="[email protected]"
@@ -24,15 +54,12 @@ RUN set -x \
         postgresql-client \
         libpq-dev \
         build-essential \
-        rustc \
-        cargo \
         chromium \
         tar \
 	jq \
         chromium-driver \
         && apt-get clean \
-    && rm -rf /var/lib/apt/lists/* \
-    && curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
+    && rm -rf /var/lib/apt/lists/*
 
 # Install Firefox from Debian repositories for ARM64 architecture
 RUN set -x \
@@ -58,18 +85,9 @@ RUN GECKODRIVER_VERSION=$(curl -s https://api.github.com/repos/mozilla/geckodriv
 RUN firefox --version
 RUN geckodriver --version
 
-# Ensure Rust directories are writable
-RUN mkdir -p /root/.rustup/downloads /root/.cargo/registry && \
-    chmod -R 777 /root/.rustup /root/.cargo
 
-# Add rust and cargo to PATH
-ENV PATH="/root/.cargo/bin:${PATH}"
 
-# Install the specific version of Rust
-RUN set -x \
-    && rustup install 1.78.0
-RUN set -x \
-    && rustup default 1.78.0
+
 
 EXPOSE 5000
 
@@ -82,8 +100,7 @@ COPY ./setup.py .
 COPY ./scripts/ scripts/
 COPY ./keyman/ keyman/
 
-# Add rust and cargo to PATH
-ENV PATH="/usr/bin/:/root/.cargo/bin:/usr/local/bin:${PATH}"
+
 
 RUN python3 -m venv /opt/venv
 
@@ -101,10 +118,11 @@ RUN set -x \
     && /opt/venv/bin/pip install wheel \
     && /opt/venv/bin/pip install .
 
-RUN ./scripts/docker/install-go.sh
-ENV PATH="${PATH}:/usr/local/go/bin"
 RUN ./scripts/docker/install-workers-deps.sh
 
+COPY --from=golang "/root/scc" "./scc"
+COPY --from=golang "/root/scorecard/scorecard" "./scorecard"
+
 # RUN ./scripts/install/workers.sh 
 
 RUN mkdir -p repos/ logs/ /augur/facade/

scripts/docker/install-golang-deps.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+set -x
+
+# Note this
+CURRENT_DIR=$PWD;
+# 18GB
+# Install scc
+SCC_DIR="$HOME/scc"
+echo "Cloning Sloc Cloc and Code (SCC) to generate value data ..."
+# this needs to be done from source. the latest version doesnt seem to exist on the package repo
+# however, the latest version (v3.5.0) requires bumping the golang version in the Dockerfile
+git clone --depth 1 --branch v3.4.0 https://github.com/boyter/scc "$SCC_DIR"
+cd $SCC_DIR
+go build;
+echo "scc build done"
+cd $CURRENT_DIR
+# 18GB
+
+# Install scorecard
+SCORECARD_DIR="$HOME/scorecard"
+echo "Cloning OSSF Scorecard to generate scorecard data ..."
+# lock version to prevent future issues if the golang version is bumped
+git clone --depth 1 --branch v5.1.1 https://github.com/ossf/scorecard $SCORECARD_DIR
+cd $SCORECARD_DIR
+go build;
+echo "scorecard build done"
+cd $CURRENT_DIR
+
+# 16GB

scripts/docker/install-workers-deps.sh

@@ -8,30 +8,3 @@ do
 	/opt/venv/bin/pip install .
 	cd $OLD
 done
-
-# install nltk
-# taken from ./scripts/install/nltk_dictionaries.sh
-for i in stopwords punkt popular universal_tagset ; do
-	/opt/venv/bin/python -m nltk.downloader $i
-done
-
-# Note this
-CURRENT_DIR=$PWD;
-
-# Install scc
-SCC_DIR="$HOME/scc"
-echo "Cloning Sloc Cloc and Code (SCC) to generate value data ..."
-git clone https://github.com/boyter/scc "$SCC_DIR"
-cd $SCC_DIR
-go build;
-echo "scc build done"
-cd $CURRENT_DIR
-
-# Install scorecard
-SCORECARD_DIR="$HOME/scorecard"
-echo "Cloning OSSF Scorecard to generate scorecard data ..."
-git clone https://github.com/ossf/scorecard $SCORECARD_DIR
-cd $SCORECARD_DIR
-go build;
-echo "scorecard build done"
-cd $CURRENT_DIR