Update and fix FreeBSD setup on Travis-CI

.travis.yml

@@ -65,11 +65,11 @@ jobs:
       services: [docker]
       install:       .travis/$JOB install
       script:        .travis/$JOB script
-#    - stage: "Stage 1"
-#      if: type IN (push, api, cron)
-#      env: JOB=cmake-freebsd
-#      install:       .travis/$JOB-stage1 install
-#      script:        .travis/$JOB-stage1 script
+    - stage: "Stage 1"
+      if: type IN (push, api, cron)
+      env: JOB=cmake-freebsd
+      install:       .travis/$JOB-stage1 install
+      script:        .travis/$JOB-stage1 script
     - stage: "Stage 1"
       if: type IN (push, api, cron)
       env: JOB=cmake-osx
@@ -81,11 +81,11 @@ jobs:
       env: JOB=tox-bootstrapd-docker
       services: [docker]
       script:        .travis/$JOB
-#    - stage: "Stage 2"
-#      if: type IN (push, api, cron)
-#      env: JOB=cmake-freebsd
-#      install:       .travis/$JOB-stage2 install
-#      script:        .travis/$JOB-stage2 script
+    - stage: "Stage 2"
+      if: type IN (push, api, cron)
+      env: JOB=cmake-freebsd
+      install:       .travis/$JOB-stage2 install
+      script:        .travis/$JOB-stage2 script
   fast_finish: true
 
 cache:

.travis/cmake-freebsd-env.sh

@@ -1,10 +1,44 @@
 #!/bin/sh
 
-NPROC=`nproc`
+# Common variables and functions
+
+NPROC=$(nproc)
 
 SCREEN_SESSION=freebsd
 SSH_PORT=10022
 
+FREEBSD_VERSION="12.1"
+IMAGE_NAME=FreeBSD-${FREEBSD_VERSION}-RELEASE-amd64.raw
+# https://download.freebsd.org/ftp/releases/VM-IMAGES/12.1-RELEASE/amd64/Latest/
+IMAGE_SHA512="a65da6260f5f894fc86fbe1f27dad7800906da7cffaa5077f82682ab74b6dd46c4ce87158c14b726d74ca3c6d611bea3bb336164da3f1cb990550310b110da22"
+
 RUN() {
-  ssh -t -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no root@localhost -p $SSH_PORT "$@"
+  ssh -t -o ConnectionAttempts=120 -o ConnectTimeout=2 -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no root@localhost -p $SSH_PORT "$@"
+}
+
+start_vm() {
+  screen -d -m qemu-system-x86_64 -curses -m 2048 -smp $NPROC -net user,hostfwd=tcp::${SSH_PORT}-:22 -net nic "$IMAGE_NAME"
+
+  # Wait for ssh to start listening on the port
+  while ! echo "exit" | nc localhost ${SSH_PORT} | grep 'OpenSSH'; do
+    sleep 5
+  done
+
+  # Test that ssh works
+  RUN uname -a
+  RUN last
+}
+
+stop_vm()
+{
+  # Turn it off
+  # We use this contraption because for some reason `shutdown -h now` and
+  # `poweroff` result in FreeBSD not shutting down on Travis (they work on my
+  # machine though)
+  RUN "shutdown -p +5sec && sleep 30" || true
+
+  # Wait for the qemu process to terminate
+  while pgrep qemu; do
+    sleep 5
+  done
 }

.travis/cmake-freebsd-stage1

@@ -1,14 +1,14 @@
 #!/bin/sh
 
+# Download and initial setup of the FreeBSD VM
+
 ACTION="$1"
 
 set -eux
 
 . .travis/cmake-freebsd-env.sh
 
 travis_install() {
-  . .travis/cmake-freebsd-install.sh
-
   git tag -l --sort=version:refname > GIT_TAGS
 
   OLD_PWD="$PWD"
@@ -18,83 +18,55 @@ travis_install() {
 
   # === Get the VM image, set it up and cache ===
 
-  # Create image if it's not cached or if this build script has changed
-  sha256sum "$OLD_PWD/.travis/cmake-freebsd-env.sh"      > /tmp/sha
-  sha256sum "$OLD_PWD/.travis/cmake-freebsd-install.sh" >> /tmp/sha
-  sha256sum "$OLD_PWD/.travis/cmake-freebsd-stage1"     >> /tmp/sha
-  if [ ! -f "./$IMAGE_NAME.tgz" ] || [ ! -f ./cmake-freebsd-stage1-all.sha256 ] || [ "`cat cmake-freebsd-stage1-all.sha256`" != "`cat /tmp/sha`" ]; then
-
+  # Create image if it's not cached, or if this build script has changed, or a new toxcore tag was pushed
+  sha256sum "$OLD_PWD/.travis/cmake-freebsd-env.sh"         > /tmp/sha
+  sha256sum "$OLD_PWD/.travis/cmake-freebsd-stage1"        >> /tmp/sha
+  sha256sum "$OLD_PWD/.travis/cmake-freebsd-stage1.expect" >> /tmp/sha
+  if [ ! -f "./$IMAGE_NAME.tgz" ] || [ ! -f ./cmake-freebsd-stage1-all.sha256 ] || [ "$(cat cmake-freebsd-stage1-all.sha256)" != "$(cat /tmp/sha)" ] || ! diff -u ./GIT_TAGS "$OLD_PWD/GIT_TAGS"; then
     rm -rf ./*
 
-    # https://download.freebsd.org/ftp/releases/VM-IMAGES/11.2-RELEASE/amd64/Latest/
-    DL_SHA512="0c3c232c7023c5036daeb5fbf68c2ddecf9703c74e317afcf19da91e83d0afcc526785571e2868894ce15cdb56b74fafa1ce9fd216469db91e021ac2ef8911e5"
-    # Selecting random mirror from https://www.freebsd.org/doc/handbook/mirrors-ftp.html
-    # Note that not all mirrors listed on that page are working, so we have removed them
-    # I'm so sorry, there are no arrays in sh and we are not using bash...
-    DL_MIRROR_1=1
-    DL_MIRROR_2=4
-    # There are 2 mirrors
-    DL_MIRROR_RANDOM=`expr $(date +%s) % 2 + 1`
-    DL_URL="ftp://ftp$(eval echo \$DL_MIRROR_$DL_MIRROR_RANDOM).us.freebsd.org/pub/FreeBSD/releases/VM-IMAGES/${FREEBSD_VERSION}-RELEASE/amd64/Latest/${IMAGE_NAME}.xz"
-
-    wget "$DL_URL"
-
-    if ! ( echo "$DL_SHA512  $IMAGE_NAME.xz" | sha512sum -c --status - ) ; then
-        echo "Error: sha512 of $IMAGE_NAME.xz doesn't match the known one"
-        exit 1
+    while true; do
+      # Selecting random mirror from https://www.freebsd.org/doc/handbook/mirrors-ftp.html
+      # Note that not all mirrors listed on that page are working, so we have removed them
+      # There are no arrays in sh so we get a bit clever
+      DL_MIRROR_1=1
+      DL_MIRROR_2=2
+      DL_MIRROR_3=3
+      DL_MIRROR_4=4
+      DL_MIRROR_5=5
+      DL_MIRROR_6=6
+      DL_MIRROR_7=7
+      DL_MIRROR_8=10
+      DL_MIRROR_9=11
+      DL_MIRROR_10=13
+      DL_MIRROR_11=14
+
+      # There are 11 mirrors
+      DL_MIRROR_RANDOM=`expr $(date +%s) % 11 + 1`
+      DL_URL="ftp://ftp$(eval echo \$DL_MIRROR_$DL_MIRROR_RANDOM).us.freebsd.org/pub/FreeBSD/releases/VM-IMAGES/${FREEBSD_VERSION}-RELEASE/amd64/Latest/${IMAGE_NAME}.xz"
+
+      # Make sure there are no partial downloads from the previous loop iterations
+      rm -rf ./*
+
+      wget --tries 1 "$DL_URL" && break
+    done
+
+    if ! ( echo "$IMAGE_SHA512  $IMAGE_NAME.xz" | sha512sum -c --status - ) ; then
+      echo "Error: sha512 of $IMAGE_NAME.xz doesn't match the known one"
+      exit 1
     fi
 
     unxz "$IMAGE_NAME.xz"
 
-    # With this we don't have to guess how long a command will run for and sleeping
-    # for that amount of time, risking either under sleeping or over sleeping, instead
-    # we will sleep exactly until the command is finished by printing out a unique
-    # string after the command is executed and then checking if it was printed.
-    execute_shell_and_wait()
-    {
-      # $RANDOM is a bash built-in, so we try to avoid name collision here by using ugly RANDOM_STR name
-      RANDOM_STR=$(< /dev/urandom tr -dc _A-Za-z0-9 | head -c16)
-      send_keys "$1;echo $RANDOM_STR
-
-"
-      # \[1B is a control escape sequence for a new line in the terminal.
-      # We want to wait for <new-line>$RANDOM_STR instead of just $RANDOM_STR because
-      # $RANDOM_STR we have inputted with send_keys above would appear in the screenlog.0
-      # file and we don't want to match our input, we want to match the echo's output.
-      # The .\? optionally matches any character. Sometimes it happens that there is some
-      # random character inserved between the new line control escape sequence and $RANDOM_STR.
-      wait_for "\[1B.\?$RANDOM_STR"
-    }
-
-    start_vm
-
-    # Login as root user
-    send_keys 'root
-
-'
-
-    # Wait for the prompt
-    wait_for "root@.*:~"
+    sudo apt-get update
+    sudo apt-get install -y qemu screen expect
 
-    # Configure network, ssh and start changing password
-    execute_shell_and_wait 'echo "ifconfig_em0=DHCP"          >> /etc/rc.conf'
-    execute_shell_and_wait 'echo "Port 22"                    >> /etc/ssh/sshd_config'
-    execute_shell_and_wait 'echo "PermitRootLogin yes"        >> /etc/ssh/sshd_config'
-    execute_shell_and_wait 'echo "PasswordAuthentication yes" >> /etc/ssh/sshd_config'
-    execute_shell_and_wait 'echo "PermitEmptyPasswords yes"   >> /etc/ssh/sshd_config'
-    execute_shell_and_wait 'echo "sshd_enable=YES"            >> /etc/rc.conf'
-    send_keys 'sh /etc/rc.d/netif restart && sh /etc/rc.d/sshd start && passwd
-'
+    # The downloaded image has little free disk space
+    qemu-img resize -f raw "$IMAGE_NAME" +5G
 
-    # Wait for the password prompt
-    wait_for "Changing local password for root"
+    NPROC=$NPROC SSH_PORT=$SSH_PORT IMAGE_NAME="$IMAGE_NAME" screen "$OLD_PWD/.travis/cmake-freebsd-stage1.expect"
 
-    # Reset password to empty for the passwordless ssh to work
-    send_keys '
-'
-    wait_for "New Password"
-    send_keys '
-'
+    start_vm
 
     # Update system
     RUN env PAGER=cat env ASSUME_ALWAYS_YES=YES freebsd-update --not-running-from-cron fetch
@@ -117,7 +89,6 @@ travis_install() {
                                                     gmake \
                                                     cmake \
                                                     pkgconf \
-                                                    opencv \
                                                     portaudio \
                                                     libsndfile \
                                                     texinfo \
@@ -130,60 +101,11 @@ travis_install() {
     # Create cache
     tar -Sczvf "$IMAGE_NAME.tgz" "$IMAGE_NAME"
     rm "$IMAGE_NAME"
-    rm screenlog.0
-
-    cp "$OLD_PWD/GIT_TAGS" .
-    sha256sum "$OLD_PWD/.travis/cmake-freebsd-env.sh"      > cmake-freebsd-stage1-all.sha256
-    sha256sum "$OLD_PWD/.travis/cmake-freebsd-install.sh" >> cmake-freebsd-stage1-all.sha256
-    sha256sum "$OLD_PWD/.travis/cmake-freebsd-stage1"     >> cmake-freebsd-stage1-all.sha256
-
-    ls -lh
-  fi
-
-  # === Update the image on new version (tag) of toxcore ===
-
-  if ! diff -u ./GIT_TAGS "$OLD_PWD/GIT_TAGS" ; then
-    # Extract the cached image
-    tar -Sxzvf "$IMAGE_NAME.tgz"
-
-    start_vm
-
-    # Log in.
-    # Although qemu prints "login:" and "Password:" lines, they are not written into the screen's log
-    # file for some reason (perhaps not enough text to flush the buffer to a file?), so we can't use
-    # wait_for on them, we just use sleep to add hopefully enough delay.
-    sleep 5
-    # "login:"
-    send_keys 'root
-
-'
-    sleep 5
-    # "Password:"
-    send_keys '
-
-'
-    # Wait for the prompt
-    wait_for "root@.* ~"
-
-    # Update system
-    RUN PAGER=cat ASSUME_ALWAYS_YES=YES freebsd-update --not-running-from-cron fetch
-    RUN PAGER=cat ASSUME_ALWAYS_YES=YES freebsd-update --not-running-from-cron install || true
-
-    # Update packages
-    RUN PAGER=cat ASSUME_ALWAYS_YES=YES pkg upgrade
-
-    # === Cache the updated VM image ===
-
-    stop_vm
-
-    # Create/Update cache
-    rm "$IMAGE_NAME.tgz"
-    tar -Sczvf "$IMAGE_NAME.tgz" "$IMAGE_NAME"
-    rm "$IMAGE_NAME"
-    rm screenlog.0
 
     cp "$OLD_PWD/GIT_TAGS" .
-
+    sha256sum "$OLD_PWD/.travis/cmake-freebsd-env.sh"         > cmake-freebsd-stage1-all.sha256
+    sha256sum "$OLD_PWD/.travis/cmake-freebsd-stage1"        >> cmake-freebsd-stage1-all.sha256
+    sha256sum "$OLD_PWD/.travis/cmake-freebsd-stage1.expect" >> cmake-freebsd-stage1-all.sha256
     ls -lh
   fi