Merge pull request davegill#6 from kkeene44/update_for_jenkins

Updated files in the "regression+feature" branch to the latest containers available from kkeene44/wrf-coop:version16, using an updated Dockerfile-first_part file, and to use the wrf-model github repositories, instead of from davegill. I also updated the maintainer to Kelly W. The following files were updated:

M    Dockerfile
M    Dockerfile-first_part
M    Dockerfile-second_part
M    Dockerfile-sed
M    README_add_feature_test.md
M    README_user.md

Dockerfile

@@ -2,18 +2,18 @@
 FROM kkeene44/wrf-coop:version16
 MAINTAINER Kelly Werner <[email protected]>
 
-RUN git clone _FORK_/_REPO_.git WRF \
+RUN git clone --recurse-submodule _FORK_/_REPO_.git WRF \
   && cd WRF \
   && git checkout _BRANCH_ \
   && cd ..
 
-RUN git clone https://github.com/davegill/SCRIPTS.git SCRIPTS \
+RUN git clone https://github.com/wrf-model/SCRIPTS.git SCRIPTS \
   && cp SCRIPTS/rd_l2_norm.py . && chmod 755 rd_l2_norm.py \
   && cp SCRIPTS/script.csh .    && chmod 755 script.csh    \
   && cp SCRIPTS/da_builds.csh . && chmod 755 da_builds.csh \
   && ln -sf SCRIPTS/Namelists . 
 
-RUN git clone https://github.com/davegill/wrf_feature_testing.git wrf_feature_testing \
+RUN git clone https://github.com/wrf-model/wrf_feature_testing.git wrf_feature_testing \
   && cd wrf_feature_testing && mv * .. && cd ..
 
 VOLUME /wrf

Dockerfile-first_part

@@ -1,15 +1,15 @@
 #
 FROM centos:7
-MAINTAINER Dave Gill <gill@ucar.edu>
+MAINTAINER Kelly Werner <kkeene@ucar.edu>
 
-ENV NML_VERSION 4.2
+ENV NML_VERSION 4.4
 
 # Set up base OS environment
 
 RUN yum -y update
 # Install SCL release package and python SCL
 RUN yum -y install centos-release-scl \
- && yum -y install --setopt=tsflags=nodocs rh-python36
+ && yum -y install --setopt=tsflags=nodocs rh-python38
 # Enable rh-python scl binary
 COPY entrypoint.sh /usr/bin/entrypoint.sh
 RUN chmod +x /usr/bin/entrypoint.sh
@@ -18,15 +18,15 @@ ENTRYPOINT [ "/usr/bin/entrypoint.sh" ]
 RUN yum -y install file gcc gcc-gfortran gcc-c++ glibc.i686 libgcc.i686 libpng-devel jasper \
   jasper-devel hostname m4 make perl tar bash tcsh time wget which zlib zlib-devel \
   openssh-clients openssh-server net-tools fontconfig libgfortran libXext libXrender \
-  sudo epel-release git flex flex-devel byacc
+  sudo epel-release git flex flex-devel byacc bzip2-devel gzip-devel
 
-# Newer version 9 of GNU compiler, required to fix WRF RRTMG-derivative builds
+# Install version 10 of GNU compiler
 
 RUN yum -y install centos-release-scl \
- && yum -y install devtoolset-9 \
- && yum -y install devtoolset-9-gcc devtoolset-9-gcc-gfortran devtoolset-9-gcc-c++ \
- && scl enable devtoolset-9 bash \
- && scl enable devtoolset-9 tcsh 
+ && yum -y install devtoolset-10 \
+ && yum -y install devtoolset-10-gcc devtoolset-10-gcc-gfortran devtoolset-10-gcc-c++ \
+ && scl enable devtoolset-10 bash \
+ && scl enable devtoolset-10 tcsh 
 
 RUN yum -y install php-devel php-pear
 RUN yum -y install ImageMagick ImageMagick-devel
@@ -42,7 +42,7 @@ ENV J 4
 
 # Build OpenMPI
 RUN mkdir -p /wrf/libs/openmpi/BUILD_DIR
-RUN source /opt/rh/devtoolset-9/enable \
+RUN source /opt/rh/devtoolset-10/enable \
  && cd /wrf/libs/openmpi/BUILD_DIR \
  && curl -L -O https://download.open-mpi.org/release/open-mpi/v4.0/openmpi-4.0.0.tar.gz \
  && tar -xf openmpi-4.0.0.tar.gz \
@@ -58,69 +58,80 @@ RUN source /opt/rh/devtoolset-9/enable \
 
 # Build HDF5 libraries
 RUN mkdir -p /wrf/libs/hdf5/BUILD_DIR
-RUN source /opt/rh/devtoolset-9/enable \
+RUN source /opt/rh/devtoolset-10/enable \
  && cd /wrf/libs/hdf5/BUILD_DIR \
- && git clone https://bitbucket.hdfgroup.org/scm/hdffv/hdf5.git \
- && cd hdf5 \
- && git checkout hdf5-1_10_4 \
+ && wget https://www2.mmm.ucar.edu/people/duda/files/mpas/sources/hdf5-1.10.5.tar.bz2 \
+ && tar -xf hdf5-1.10.5.tar.bz2 \
+ && cd hdf5-1.10.5 \
  && ./configure --enable-fortran --enable-cxx --enable-shared --prefix=/usr/local/ &> /wrf/libs/build_log_hdf5_config \
  && echo dummy printout to keep travis happy hdf5 config \
  && make install &> /wrf/libs/build_log_hdf5_make \
  && echo dummy printout to keep travis happy hdf5 make \
+ && cd / \
  && rm -rf /wrf/libs/hdf5/BUILD_DIR
 ENV LD_LIBRARY_PATH /usr/local/lib
 
 # Build netCDF C libraries
 RUN yum -y install libcurl-devel zlib-devel
 ENV NETCDF /wrf/libs/netcdf
 RUN mkdir -p ${NETCDF}/BUILD_DIR
-RUN source /opt/rh/devtoolset-9/enable \
+RUN source /opt/rh/devtoolset-10/enable \
  && cd ${NETCDF}/BUILD_DIR \
- && curl -L -O https://github.com/Unidata/netcdf-c/archive/v4.6.2.tar.gz \
- && curl -L -O https://github.com/Unidata/netcdf-fortran/archive/v4.4.5.tar.gz \
+ && curl -L -O https://github.com/Unidata/netcdf-c/archive/refs/tags/v4.7.4.tar.gz \
+ && curl -L -O https://github.com/Unidata/netcdf-fortran/archive/refs/tags/v4.6.0.tar.gz \
  && curl -L -O https://github.com/Unidata/netcdf4-python/archive/v1.5.3rel.tar.gz \
- && tar -xf v4.6.2.tar.gz \
- && cd netcdf-c-4.6.2 \
+ && tar -xf v4.7.4.tar.gz \
+ && cd netcdf-c-4.7.4 \
  && ./configure --enable-shared --prefix=${NETCDF} &> /wrf/libs/build_log_ncc_config \
  && echo dummy printout to keep travis happy ncc config \
  && make install &> /wrf/libs/build_log_ncc_make \
- && echo dummy printout to keep travis happy ncc make
+ && echo dummy printout to keep travis happy ncc make \
+ && cd /
 
 # Build netCDF Fortran libraries
-RUN source /opt/rh/devtoolset-9/enable \
+RUN source /opt/rh/devtoolset-10/enable \
  && env \
  && cd ${NETCDF}/BUILD_DIR \
- && tar -xf v4.4.5.tar.gz \
- && cd netcdf-fortran-4.4.5/ \
- && export LD_LIBRARY_PATH=${NETCDF}/lib:${LD_LIBRARY_PATH} \
+ && tar -xf v4.6.0.tar.gz \
+ && cd netcdf-fortran-4.6.0/ \
+ && export LD_LIBRARY_PATH=${NETCDF}/lib:/usr/local/lib:${LD_LIBRARY_PATH} \
  && CPPFLAGS=-I${NETCDF}/include LDFLAGS=-L${NETCDF}/lib ./configure --enable-shared --prefix=${NETCDF} &> /wrf/libs/build_log_ncf_config \
  && echo dummy printout to keep travis happy ncf config \
  && make install &> /wrf/libs/build_log_ncf_make \
- && echo dummy printout to keep travis happy ncf make
-
-# Setup up python3.6
-RUN yum -y install python36 \
- && yum -y install python36-devel \
- && yum -y install python36-pip \
- && yum -y install python36-setuptools \
- && easy_install-3.6 pip \
- && python3.6 -m pip install numpy \
- && python3.6 -m pip install --upgrade pip \
- && python3.6 -m pip install --upgrade setuptools
+ && echo dummy printout to keep travis happy ncf make \
+ && cd /
+
+# Install and setup Python3.8
+RUN yum -y groupinstall "Development Tools"
+RUN yum -y install openssl-devel bzip2-devel libffi-devel xz-devel
+RUN mkdir -p /wrf/libs/python/BUILD_DIR \
+  && cd /wrf/libs/python/BUILD_DIR \
+  && wget https://www.python.org/ftp/python/3.8.12/Python-3.8.12.tgz \
+  && tar -xf Python-3.8.12.tgz \
+  && cd Python-3.8.12 \
+  && ./configure --enable-optimizations --prefix=/usr/local  \
+  && make altinstall \
+  && cd / \
+  && rm -rf /wrf/libs/python/BUILD_DIR
+RUN python3.8 -m pip install numpy \
+ && python3.8 -m pip install --upgrade pip \
+ && python3.8 -m pip install --upgrade setuptools
 RUN ldconfig -v
 
 # Build netCDF4-python libraries
-RUN source /opt/rh/devtoolset-9/enable \
+RUN source /opt/rh/devtoolset-10/enable \
  && cd ${NETCDF}/BUILD_DIR \
  && tar -xf v1.5.3rel.tar.gz \
  && cd netcdf4-python-1.5.3rel/ \
  && export LD_LIBRARY_PATH=${NETCDF}/lib:${LD_LIBRARY_PATH} \
  && export NETCDF4_DIR=${NETCDF} \
  && export HDF5_DIR=/usr/local \
- && python3.6 -m pip install wheel \
- && python3.6 setup.py build \
- && CPPFLAGS="-I${NETCDF}/include -I/usr/local/include" LDFLAGS="-L${NETCDF}/lib -L/usr/local/lib" python3.6 setup.py install \
- && echo dummy printout to keep travis happy ncf4-python install
+ && python3.8 -m pip install wheel \
+ && python3.8 setup.py build \
+ && CPPFLAGS="-I${NETCDF}/include -I/usr/local/include" LDFLAGS="-L${NETCDF}/lib -L/usr/local/lib" python3.8 setup.py install \
+ && echo dummy printout to keep travis happy ncf4-python install \
+ && cd / \
+ && rm -rf ${NETCDF}/BUILD_DIR
 
 RUN mkdir -p /var/run/sshd \
     && ssh-keygen -A \
@@ -133,24 +144,25 @@ RUN mkdir -p  /wrf/WPS_GEOG /wrf/wrfinput /wrf/wrfoutput \
  &&  chmod 6755 /wrf /wrf/WPS_GEOG /wrf/wrfinput /wrf/wrfoutput /usr/local
 
 # Download NCL
-#https://www.earthsystemgrid.org/dataset/ncl.630.1.html
-RUN curl -SL https://www.earthsystemgrid.org/api/v1/dataset/ncl.630.1/file/ncl_ncarg-6.3.0.Linux_CentOS7.0_x86_64_nodap_gcc482.tar.gz | tar zxC /usr/local
+RUN curl -SL https://www.earthsystemgrid.org/api/v1/dataset/ncl.662.dap/file/ncl_ncarg-6.6.2-CentOS7.6_64bit_gnu485.tar.gz | tar zxC /usr/local 
 ENV NCARG_ROOT /usr/local
 
 # Set environment for interactive container shells
 RUN echo export LDFLAGS="-lm" >> /etc/bashrc \
  && echo export NETCDF=${NETCDF} >> /etc/bashrc \
  && echo export JASPERINC=/usr/include/jasper/ >> /etc/bashrc \
  && echo export JASPERLIB=/usr/lib64/ >> /etc/bashrc \
- && echo export LD_LIBRARY_PATH="/opt/rh/devtoolset-9/root/usr/lib/gcc/x86_64-redhat-linux/9:/usr/lib64/openmpi/lib:${NETCDF}/lib:${LD_LIBRARY_PATH}" >> /etc/bashrc  \
- && echo export PATH=".:/opt/rh/devtoolset-9/root/usr/bin:/usr/lib64/openmpi/bin:${NETCDF}/bin:$PATH" >> /etc/bashrc
+ && echo export LD_LIBRARY_PATH="/opt/rh/devtoolset-10/root/usr/lib/gcc/aarch64-redhat-linux/10:/usr/lib64/openmpi/lib:${NETCDF}/lib:${LD_LIBRARY_PATH}" >> /etc/bashrc  \
+ && echo export PATH=".:/opt/rh/devtoolset-10/root/usr/bin:/usr/lib64/openmpi/bin:${NETCDF}/bin:$PATH" >> /etc/bashrc \
+ && echo export NCARG_ROOT=/usr/local >> /etc/bashrc
 
 RUN echo setenv LDFLAGS "-lm" >> /etc/csh.cshrc \
  && echo setenv NETCDF "${NETCDF}" >> /etc/csh.cshrc \
  && echo setenv JASPERINC "/usr/include/jasper/" >> /etc/csh.cshrc \
  && echo setenv JASPERLIB "/usr/lib64/" >> /etc/csh.cshrc \
- && echo setenv LD_LIBRARY_PATH "/opt/rh/devtoolset-9/root/usr/lib/gcc/x86_64-redhat-linux/9:/usr/lib64/openmpi/lib:${NETCDF}/lib:${LD_LIBRARY_PATH}" >> /etc/csh.cshrc \
- && echo setenv PATH ".:/opt/rh/devtoolset-9/root/usr/bin:/usr/lib64/openmpi/bin:${NETCDF}/bin:$PATH" >> /etc/csh.cshrc
+ && echo setenv LD_LIBRARY_PATH "/opt/rh/devtoolset-10/root/usr/lib/gcc/aarch64-redhat-linux/10:/usr/lib64/openmpi/lib:${NETCDF}/lib:${LD_LIBRARY_PATH}" >> /etc/csh.cshrc \
+ && echo setenv PATH ".:/opt/rh/devtoolset-10/root/usr/bin:/usr/lib64/openmpi/bin:${NETCDF}/bin:$PATH" >> /etc/csh.cshrc \
+ && echo setenv NCARG_ROOT /usr/local >> /etc/csh.cshrc
 
 RUN mkdir /wrf/.ssh ; echo "StrictHostKeyChecking no" > /wrf/.ssh/config
 COPY default-mca-params.conf /wrf/.openmpi/mca-params.conf
@@ -169,13 +181,12 @@ RUN if [ "$argname" = "tutorial" ] ; then curl -SL https://www2.mmm.ucar.edu/wrf
 RUN if [ "$argname" = "tutorial" ] ; then curl -SL https://www2.mmm.ucar.edu/wrf/src/namelists_v$NML_VERSION.tar.gz  | tar -xzC /wrf/wrfinput ; fi
 RUN if [ "$argname" = "tutorial" ] ; then curl -SL https://www2.mmm.ucar.edu/wrf/TUTORIAL_DATA/WRF_NCL_scripts.tar.gz | tar -xzC /wrf ; fi
 RUN if [ "$argname" = "regtest" ]  ; then curl -SL https://www2.mmm.ucar.edu/wrf/dave/data_smaller.tar.gz | tar -xzC /wrf ; fi
-RUN if [ "$argname" = "regtest" ]  ; then curl -SL https://www2.mmm.ucar.edu/wrf/dave/feature_data.tar.gz | tar -xzC /wrf ; fi
 
 ENV JASPERINC /usr/include/jasper
 ENV JASPERLIB /usr/lib64
 ENV NETCDF_classic 1
-ENV LD_LIBRARY_PATH /opt/rh/devtoolset-9/root/usr/lib/gcc/x86_64-redhat-linux/9:/usr/lib64/openmpi/lib:${NETCDF}/lib:${LD_LIBRARY_PATH}
-ENV PATH  .:/opt/rh/devtoolset-9/root/usr/bin:/usr/lib64/openmpi/bin:${NETCDF}/bin:$PATH
+ENV LD_LIBRARY_PATH /opt/rh/devtoolset-10/root/usr/lib/gcc/aarch64-redhat-linux/10:/usr/lib64/openmpi/lib:${NETCDF}/lib:${LD_LIBRARY_PATH}
+ENV PATH  .:/opt/rh/devtoolset-10/root/usr/bin:/usr/lib64/openmpi/bin:${NETCDF}/bin:$PATH
 
 RUN ssh-keygen -f /wrf/.ssh/id_rsa -t rsa -N '' \
     && chmod 600 /wrf/.ssh/config \

Dockerfile-second_part

@@ -1,28 +1,18 @@
 #
-FROM davegill/wrf-coop:fifteenthtry
-MAINTAINER Dave Gill <gill@ucar.edu>
+FROM kkeene44/wrf-coop:version16
+MAINTAINER Kelly Werner <kkeene@ucar.edu>
 
-#RUN echo _HERE1_
-#RUN git clone https://github.com/davegill/WRF.git davegill/WRF \
-#  && cd davegill/WRF \
-#  && git fetch origin +refs/pull/4/merge: \
-#  && git checkout -qf FETCH_HEAD \
-#  && cd .. \
-#  && mv WRF /wrf/WRF
-#RUN echo _HERE2_
-
-RUN git clone https://github.com/wrf-model/WRF.git WRF \
+RUN git clone --recurse-submodule https://github.com/wrf-model/WRF.git WRF \
   && cd WRF \
-  && git checkout release-v4.2.1 \
+  && git checkout release-v4.4.2 \
   && git checkout master \
   && git checkout develop \
-  && git checkout release-v4.2 \
+  && git checkout release-v4.4.1 \
   && cd ..
 
-RUN git clone https://github.com/davegill/SCRIPTS.git SCRIPTS \
+RUN git clone https://github.com/wrf-model/SCRIPTS.git SCRIPTS \
   && cp SCRIPTS/rd_l2_norm.py . && chmod 755 rd_l2_norm.py \
   && cp SCRIPTS/script.csh .    && chmod 755 script.csh    \
-  && cp SCRIPTS/da_builds.csh . && chmod 755 da_builds.csh \
   && ln -sf SCRIPTS/Namelists . 
 
 VOLUME /wrf

Dockerfile-sed

@@ -1,22 +1,22 @@
 #
-FROM davegill/wrf-coop:fifteenthtry
-MAINTAINER Dave Gill <gill@ucar.edu>
+FROM kkeene44/wrf-coop:version16
+MAINTAINER Kelly Werner <kkeene@ucar.edu>
 
 #	DO NOT CHANGE UNDERSCORE GIT STRINGS
 #	THESE ARE USED FOR JENKINS TESTING
 
-RUN git clone _GIT_URL_ WRF \
+RUN git clone --recurse-submodule _GIT_URL_ WRF \
   && cd WRF \
   && git checkout _GIT_BRANCH_ \
   && cd ..
 
-RUN git clone https://github.com/davegill/SCRIPTS.git SCRIPTS \
+RUN git clone https://github.com/wrf-model/SCRIPTS.git SCRIPTS \
   && cp SCRIPTS/rd_l2_norm.py . && chmod 755 rd_l2_norm.py \
   && cp SCRIPTS/script.csh .    && chmod 755 script.csh    \
   && cp SCRIPTS/da_builds.csh . && chmod 755 da_builds.csh \
   && ln -sf SCRIPTS/Namelists . 
 
-RUN git clone https://github.com/davegill/wrf_feature_testing.git wrf_feature_testing \
+RUN git clone https://github.com/wrf-model/wrf_feature_testing.git wrf_feature_testing \
   && cd wrf_feature_testing && mv * .. && cd ..
 
 VOLUME /wrf

README_add_feature_test.md

@@ -38,7 +38,7 @@ No other changes are required in the build.csh file.
 ### cases/adapt
 
 The second change is in a separate repository. 
-https://github.com/davegill/wrf_feature_testing.
+https://github.com/wrf-model/wrf_feature_testing.
 The `cases` directory has the name of each of the required feature tests. 
 
 A new directory is added, with three files. The directory name, in this instance,

README_user.md

@@ -72,8 +72,8 @@ Several types of tests are accessible within this docker testing system.
 |ARW 2D hill         |     4     |   2D  |  yes   |        |     |   ideal    | N |
 
 2. The testing uses the WRF run-time configuration file, `namelist.input` to exercise an expandable list of features that are all included within the WRF docker container. The current list of tests conducted is produced from information within two github respositories:
-   * All available namelists choices for em_real: https://github.com/davegill/SCRIPTS/tree/master/Namelists/weekly/em_real/MPI
-   * Requested tests are defined in: https://github.com/davegill/wrf-coop/blob/regression+feature/build.csh
+   * All available namelists choices for em_real: https://github.com/wrf-model/SCRIPTS/tree/master/Namelists/weekly/em_real/MPI
+   * Requested tests are defined in: https://github.com/wrf-model/wrf-coop/blob/regression+feature/build.csh
 
 | **Test** | **MP** | **CU** | **LW** | **SW** | **PBL** | **SFC** | **LSM** | **URB** |
 | ------|:--:|:--:|:--:|:--:|:--: |:--: |:--: |:--: |
@@ -128,8 +128,8 @@ The topography for the nested domains over the central US for the 30/10-km ARW r
 Currrently all of the restart builds are for ARW em_real. Since the comparison is between the first (the full-length simulation) and the second (shorter, restart simulation) WRF runs, there is no need to try out different parallel options. The time period is 2016 Mar 23-24 0000 UTC, though again the simulations are very short: 12 minutes in duration.
 
 The testing uses groupings of three WRF run-time configuration files, `namelist.input.1`, `namelist.input.3`, and `namelist.input.3` to exercise an expandable list of features that are all included within the WRF docker container. The current list of tests conducted is produced from information within two githhub respositories:
-   * All available namelists choices for em_real: https://github.com/davegill/wrf_feature_testing/tree/main/cases
-   * Requested tests are defined in: https://github.com/davegill/wrf-coop/blob/regression+feature/build.csh
+   * All available namelists choices for em_real: https://github.com/wrf-model/wrf_feature_testing/tree/main/cases
+   * Requested tests are defined in: https://github.com/wrf-model/wrf-coop/blob/regression+feature/build.csh
 
 
 | **Test** | **SUITE** | **URB** | **DFI** |
@@ -152,7 +152,7 @@ The following describe the required steps run the WRF regression system on you l
 
 2. To start the process of constructing a working WRF docker container, clone the WRF-specific wrf-coop repository, and checkout the specific branch used by the automated testing. Once you have the docker application running on your machine, this repository contains the code that eventually builds the container structures for WRF.
 ```
-git clone https://github.com/davegill/wrf-coop
+git clone https://github.com/wrf-model/wrf-coop
 cd wrf-coop
 git checkout regression+feature
 ```
@@ -164,20 +164,20 @@ git checkout regression+feature
 Here is the entire Dockerfile for ARW: `Dockerfile`:
 ```
 #
-FROM davegill/wrf-coop:fifteenththtry
-MAINTAINER Dave Gill <gill@ucar.edu>
+FROM kkeene44/wrf-coop:version16
+MAINTAINER Kelly Werner <kkeene@ucar.edu>
 
 RUN git clone _FORK_/_REPO_.git WRF \
   && cd WRF \
   && git checkout _BRANCH_ \
   && cd ..
 
-RUN git clone https://github.com/davegill/SCRIPTS.git SCRIPTS \
+RUN git clone https://github.com/wrf-model/SCRIPTS.git SCRIPTS \
   && cp SCRIPTS/rd_l2_norm.py . && chmod 755 rd_l2_norm.py \
   && cp SCRIPTS/script.csh .    && chmod 755 script.csh    \
   && ln -sf SCRIPTS/Namelists . 
   
-RUN git clone https://github.com/davegill/wrf_feature_testing.git wrf_feature_testing \
+RUN git clone https://github.com/wrf-model/wrf_feature_testing.git wrf_feature_testing \
   && cd wrf_feature_testing && mv * .. && cd ..
 
 VOLUME /wrf