From 2c4a761f487414798d2d65f3deb1bd7ba0936ee7 Mon Sep 17 00:00:00 2001 From: zikun <33176974+zikun@users.noreply.github.com> Date: Tue, 9 Jun 2020 12:20:29 +0800 Subject: [PATCH 1/4] Add ADDITONAL_DEV_DEPS and ADDITONAL_RUNTIME_DEPS --- Dockerfile | 12 ++++++++++-- Dockerfile.ci | 8 ++++++++ 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index fc9c153446fa9..1cad760f8b835 100644 --- a/Dockerfile +++ b/Dockerfile @@ -79,7 +79,10 @@ RUN apt-get update \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* -# Install basic apt dependencies +ARG ADDITONAL_DEV_DEPS="" +ENV ADDITONAL_DEV_DEPS=${ADDITONAL_DEV_DEPS} + +# Install basic and additional apt dependencies RUN curl --fail --location https://deb.nodesource.com/setup_10.x | bash - \ && curl https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - > /dev/null \ && echo "deb https://dl.yarnpkg.com/debian/ stable main" > /etc/apt/sources.list.d/yarn.list \ @@ -120,6 +123,7 @@ RUN curl --fail --location https://deb.nodesource.com/setup_10.x | bash - \ unixodbc \ unixodbc-dev \ yarn \ + ${ADDITONAL_DEV_DEPS} \ && apt-get autoremove -yqq --purge \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* @@ -241,13 +245,16 @@ ENV PYTHON_BASE_IMAGE=${PYTHON_BASE_IMAGE} ARG AIRFLOW_VERSION ENV AIRFLOW_VERSION=${AIRFLOW_VERSION} +ARG ADDITONAL_RUNTIME_DEPS="" +ENV ADDITONAL_RUNTIME_DEPS=${ADDITONAL_RUNTIME_DEPS} + # Make sure noninteractive debian install is used and language variables set ENV DEBIAN_FRONTEND=noninteractive LANGUAGE=C.UTF-8 LANG=C.UTF-8 LC_ALL=C.UTF-8 \ LC_CTYPE=C.UTF-8 LC_MESSAGES=C.UTF-8 # Note missing man directories on debian-buster # https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=863199 -# Install basic apt dependencies +# Install basic and additional apt dependencies RUN mkdir -pv /usr/share/man/man1 \ && mkdir -pv /usr/share/man/man7 \ && apt-get update \ @@ -276,6 +283,7 @@ RUN mkdir -pv /usr/share/man/man1 \ sqlite3 \ sudo \ unixodbc \ + ${ADDITONAL_RUNTIME_DEPS} \ && apt-get autoremove -yqq --purge \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* diff --git a/Dockerfile.ci b/Dockerfile.ci index b467d48aada9a..fc18d08bba245 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -54,6 +54,9 @@ RUN apt-get update \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* +ARG ADDITONAL_DEV_DEPS="" +ENV ADDITONAL_DEV_DEPS=${ADDITONAL_DEV_DEPS} + # Install basic apt dependencies RUN curl --fail --location https://deb.nodesource.com/setup_10.x | bash - \ && curl https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - > /dev/null \ @@ -85,6 +88,7 @@ RUN curl --fail --location https://deb.nodesource.com/setup_10.x | bash - \ unixodbc \ unixodbc-dev \ yarn \ + ${ADDITONAL_DEV_DEPS} \ && apt-get autoremove -yqq --purge \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* @@ -117,6 +121,9 @@ RUN adduser airflow \ && echo "airflow ALL=(ALL) NOPASSWD: ALL" > /etc/sudoers.d/airflow \ && chmod 0440 /etc/sudoers.d/airflow +ARG ADDITONAL_RUNTIME_DEPS="" +ENV ADDITONAL_RUNTIME_DEPS=${ADDITONAL_RUNTIME_DEPS} + # Note missing man directories on debian-buster # https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=863199 RUN mkdir -pv /usr/share/man/man1 \ @@ -144,6 +151,7 @@ RUN mkdir -pv /usr/share/man/man1 \ tmux \ unzip \ vim \ + ${ADDITONAL_RUNTIME_DEPS} \ && apt-get autoremove -yqq --purge \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* From d4c0e1b605ac2293a8b1599495e9e0293bb90900 Mon Sep 17 00:00:00 2001 From: zikun <33176974+zikun@users.noreply.github.com> Date: Tue, 9 Jun 2020 14:27:22 +0800 Subject: [PATCH 2/4] Add examples for additional apt dev and runtime dependencies --- IMAGES.rst | 44 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/IMAGES.rst b/IMAGES.rst index e53d3cadd9e42..7e23c4d411832 100644 --- a/IMAGES.rst +++ b/IMAGES.rst @@ -203,6 +203,12 @@ The following build arguments (``--build-arg`` in docker build command) can be u | ``ADDITIONAL_PYTHON_DEPS`` | \```\` | additional python dependencies to | | | | install | +------------------------------------------+------------------------------------------+------------------------------------------+ +| ``ADDITIONAL_DEV_DEPS`` | ```` | additional apt dev dependencies to | +| | | install | ++------------------------------------------+------------------------------------------+------------------------------------------+ +| ``ADDITIONAL_RUNTIME_DEPS`` | ```` | additional apt runtime dependencies to | +| | | install | ++------------------------------------------+------------------------------------------+------------------------------------------+ Here are some examples of how CI images can built manually. CI is always built from local sources. @@ -236,6 +242,20 @@ This builds the CI image in version 3.6 with "mssql" additional package added. docker build . -f Dockerfile.ci --build-arg PYTHON_BASE_IMAGE="python:3.7-slim-buster" \ --build-arg PYTHON_MAJOR_MINOR_VERSION=3.6 --build-arg ADDITIONAL_PYTHON_DEPS="mssql" +This builds the CI image in version 3.6 with "gcc" and "g++" additional apt dev dependencies added. + +.. code-block:: + + docker build . -f Dockerfile.ci --build-arg PYTHON_BASE_IMAGE="python:3.7-slim-buster" \ + --build-arg PYTHON_MAJOR_MINOR_VERSION=3.6 --build-arg ADDITONAL_DEV_DEPS="gcc g++" + +This builds the CI image in version 3.6 with "jdbc" extra and "default-jre-headless" additional apt runtime dependencies added. + +.. code-block:: + + docker build . -f Dockerfile.ci --build-arg PYTHON_BASE_IMAGE="python:3.7-slim-buster" \ + --build-arg PYTHON_MAJOR_MINOR_VERSION=3.6 --build-arg AIRFLOW_EXTRAS=jdbc --build-arg ADDITONAL_RUNTIME_DEPS="default-jre-headless" + Production images @@ -277,6 +297,12 @@ The following build arguments (``--build-arg`` in docker build command) can be u | ``ADDITIONAL_PYTHON_DEPS`` | ```` | Optional python packages to extend | | | | the image with some extra dependencies | +------------------------------------------+------------------------------------------+------------------------------------------+ +| ``ADDITIONAL_DEV_DEPS`` | ```` | additional apt dev dependencies to | +| | | install | ++------------------------------------------+------------------------------------------+------------------------------------------+ +| ``ADDITIONAL_RUNTIME_DEPS`` | ```` | additional apt runtime dependencies to | +| | | install | ++------------------------------------------+------------------------------------------+------------------------------------------+ | ``AIRFLOW_HOME`` | ``/opt/airflow`` | Airflow’s HOME (that’s where logs and | | | | sqlite databases are stored) | +------------------------------------------+------------------------------------------+------------------------------------------+ @@ -402,6 +428,24 @@ additional python dependencies. --build-arg ADDITIONAL_AIRFLOW_EXTRAS="mssql,hdfs" --build-arg ADDITIONAL_PYTHON_DEPS="sshtunnel oauth2client" +This builds the production image in version 3.7 with additional airflow extras from 1.10.10 Pypi package and +additional apt dev and runtime dependencies. + +.. code-block:: + + docker build . \ + --build-arg PYTHON_BASE_IMAGE="python:3.7-slim-buster" \ + --build-arg PYTHON_MAJOR_MINOR_VERSION=3.7 \ + --build-arg AIRFLOW_INSTALL_SOURCES="apache-airflow" \ + --build-arg AIRFLOW_INSTALL_VERSION="==1.10.10" \ + --build-arg CONSTRAINT_REQUIREMENTS="https://raw.githubusercontent.com/apache/airflow/1.10.10/requirements/requirements-python3.7.txt" \ + --build-arg ENTRYPOINT_FILE="https://raw.githubusercontent.com/apache/airflow/1.10.10/entrypoint.sh" \ + --build-arg AIRFLOW_SOURCES_FROM="entrypoint.sh" \ + --build-arg AIRFLOW_SOURCES_TO="/entrypoint" \ + --build-arg ADDITIONAL_AIRFLOW_EXTRAS="jdbc" + --build-arg ADDITIONAL_DEV_DEPS="gcc g++" + --build-arg ADDITIONAL_RUNTIME_DEPS="default-jre-headless" + Image manifests --------------- From f535d87efcc58a79189d8a4b6a6c3d59fe2aabe3 Mon Sep 17 00:00:00 2001 From: zikun <33176974+zikun@users.noreply.github.com> Date: Tue, 9 Jun 2020 14:46:51 +0800 Subject: [PATCH 3/4] Update comment --- Dockerfile.ci | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile.ci b/Dockerfile.ci index fc18d08bba245..11a667a1e41ca 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -57,7 +57,7 @@ RUN apt-get update \ ARG ADDITONAL_DEV_DEPS="" ENV ADDITONAL_DEV_DEPS=${ADDITONAL_DEV_DEPS} -# Install basic apt dependencies +# Install basic and additional apt dependencies RUN curl --fail --location https://deb.nodesource.com/setup_10.x | bash - \ && curl https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - > /dev/null \ && echo "deb https://dl.yarnpkg.com/debian/ stable main" > /etc/apt/sources.list.d/yarn.list \ From 3e96d9a72358d9a01678fc3f1df284c34bf1b5d7 Mon Sep 17 00:00:00 2001 From: zikun <33176974+zikun@users.noreply.github.com> Date: Tue, 9 Jun 2020 21:45:40 +0800 Subject: [PATCH 4/4] Fix typo --- Dockerfile | 12 ++++++------ Dockerfile.ci | 12 ++++++------ IMAGES.rst | 4 ++-- docs/concepts.rst | 2 +- 4 files changed, 15 insertions(+), 15 deletions(-) diff --git a/Dockerfile b/Dockerfile index 1cad760f8b835..588262fbd4026 100644 --- a/Dockerfile +++ b/Dockerfile @@ -79,8 +79,8 @@ RUN apt-get update \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* -ARG ADDITONAL_DEV_DEPS="" -ENV ADDITONAL_DEV_DEPS=${ADDITONAL_DEV_DEPS} +ARG ADDITIONAL_DEV_DEPS="" +ENV ADDITIONAL_DEV_DEPS=${ADDITIONAL_DEV_DEPS} # Install basic and additional apt dependencies RUN curl --fail --location https://deb.nodesource.com/setup_10.x | bash - \ @@ -123,7 +123,7 @@ RUN curl --fail --location https://deb.nodesource.com/setup_10.x | bash - \ unixodbc \ unixodbc-dev \ yarn \ - ${ADDITONAL_DEV_DEPS} \ + ${ADDITIONAL_DEV_DEPS} \ && apt-get autoremove -yqq --purge \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* @@ -245,8 +245,8 @@ ENV PYTHON_BASE_IMAGE=${PYTHON_BASE_IMAGE} ARG AIRFLOW_VERSION ENV AIRFLOW_VERSION=${AIRFLOW_VERSION} -ARG ADDITONAL_RUNTIME_DEPS="" -ENV ADDITONAL_RUNTIME_DEPS=${ADDITONAL_RUNTIME_DEPS} +ARG ADDITIONAL_RUNTIME_DEPS="" +ENV ADDITIONAL_RUNTIME_DEPS=${ADDITIONAL_RUNTIME_DEPS} # Make sure noninteractive debian install is used and language variables set ENV DEBIAN_FRONTEND=noninteractive LANGUAGE=C.UTF-8 LANG=C.UTF-8 LC_ALL=C.UTF-8 \ @@ -283,7 +283,7 @@ RUN mkdir -pv /usr/share/man/man1 \ sqlite3 \ sudo \ unixodbc \ - ${ADDITONAL_RUNTIME_DEPS} \ + ${ADDITIONAL_RUNTIME_DEPS} \ && apt-get autoremove -yqq --purge \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* diff --git a/Dockerfile.ci b/Dockerfile.ci index 11a667a1e41ca..ce9d2775698e2 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -54,8 +54,8 @@ RUN apt-get update \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* -ARG ADDITONAL_DEV_DEPS="" -ENV ADDITONAL_DEV_DEPS=${ADDITONAL_DEV_DEPS} +ARG ADDITIONAL_DEV_DEPS="" +ENV ADDITIONAL_DEV_DEPS=${ADDITIONAL_DEV_DEPS} # Install basic and additional apt dependencies RUN curl --fail --location https://deb.nodesource.com/setup_10.x | bash - \ @@ -88,7 +88,7 @@ RUN curl --fail --location https://deb.nodesource.com/setup_10.x | bash - \ unixodbc \ unixodbc-dev \ yarn \ - ${ADDITONAL_DEV_DEPS} \ + ${ADDITIONAL_DEV_DEPS} \ && apt-get autoremove -yqq --purge \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* @@ -121,8 +121,8 @@ RUN adduser airflow \ && echo "airflow ALL=(ALL) NOPASSWD: ALL" > /etc/sudoers.d/airflow \ && chmod 0440 /etc/sudoers.d/airflow -ARG ADDITONAL_RUNTIME_DEPS="" -ENV ADDITONAL_RUNTIME_DEPS=${ADDITONAL_RUNTIME_DEPS} +ARG ADDITIONAL_RUNTIME_DEPS="" +ENV ADDITIONAL_RUNTIME_DEPS=${ADDITIONAL_RUNTIME_DEPS} # Note missing man directories on debian-buster # https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=863199 @@ -151,7 +151,7 @@ RUN mkdir -pv /usr/share/man/man1 \ tmux \ unzip \ vim \ - ${ADDITONAL_RUNTIME_DEPS} \ + ${ADDITIONAL_RUNTIME_DEPS} \ && apt-get autoremove -yqq --purge \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* diff --git a/IMAGES.rst b/IMAGES.rst index 7e23c4d411832..d789e19fd812b 100644 --- a/IMAGES.rst +++ b/IMAGES.rst @@ -247,14 +247,14 @@ This builds the CI image in version 3.6 with "gcc" and "g++" additional apt dev .. code-block:: docker build . -f Dockerfile.ci --build-arg PYTHON_BASE_IMAGE="python:3.7-slim-buster" \ - --build-arg PYTHON_MAJOR_MINOR_VERSION=3.6 --build-arg ADDITONAL_DEV_DEPS="gcc g++" + --build-arg PYTHON_MAJOR_MINOR_VERSION=3.6 --build-arg ADDITIONAL_DEV_DEPS="gcc g++" This builds the CI image in version 3.6 with "jdbc" extra and "default-jre-headless" additional apt runtime dependencies added. .. code-block:: docker build . -f Dockerfile.ci --build-arg PYTHON_BASE_IMAGE="python:3.7-slim-buster" \ - --build-arg PYTHON_MAJOR_MINOR_VERSION=3.6 --build-arg AIRFLOW_EXTRAS=jdbc --build-arg ADDITONAL_RUNTIME_DEPS="default-jre-headless" + --build-arg PYTHON_MAJOR_MINOR_VERSION=3.6 --build-arg AIRFLOW_EXTRAS=jdbc --build-arg ADDITIONAL_RUNTIME_DEPS="default-jre-headless" diff --git a/docs/concepts.rst b/docs/concepts.rst index 387cacda9b687..e016b80b106d6 100644 --- a/docs/concepts.rst +++ b/docs/concepts.rst @@ -566,7 +566,7 @@ with the same ``conn_id``, the :py:meth:`~airflow.hooks.base_hook.BaseHook.get_c provide basic load balancing and fault tolerance, when used in conjunction with retries. Airflow also provides a mechanism to store connections outside the database, e.g. in :ref:`environment variables `. -Additonal sources may be enabled, e.g. :ref:`AWS SSM Parameter Store `, or you may +Additional sources may be enabled, e.g. :ref:`AWS SSM Parameter Store `, or you may :ref:`roll your own secrets backend `. Many hooks have a default ``conn_id``, where operators using that hook do not