Enable travis pipelines (and do some minor improvements)

.travis.yml

@@ -1,69 +1,110 @@
-sudo: required
-services:
-  - docker
+sudo: false
 
 language: python
 
 python:
-  # python3.4.2 has bug in http.cookies module, aiohttp provides fix
-  - 3.4.2
-  - 3.4.3
-  - 3.5.2
-  # - 3.5.3
-  - 3.5
-  - 3.6
-  # - 3.7-dev
-  # - 'nightly'
+# python3.4.2 has bug in http.cookies module, aiohttp provides fix for it
+- 3.4.2
+- 3.4.3
+- 3.5.2
+- 3.5
+- &mainstream_python 3.6
 
-os:
-  - linux
-#   - osx   # doesn't work on MacOSX -- the system has no Python installed
+install:
+- &upgrade_python_toolset pip install --upgrade pip setuptools wheel
+- pip install -U -r requirements/ci.txt
 
-matrix:
-  allow_failures:
-    - python: '3.6-dev'
-    - python: 'nightly'
-    - os: osx
+script:
+- make cov-ci-no-ext
+- make cov-ci-aio-debug
+- make cov-ci-run
+- python setup.py check -rm
+- if python -c "import sys; sys.exit(sys.version_info < (3,6))"; then
+      python setup.py check -s;
+  fi
 
-cache:
-  directories:
-  - $HOME/.cache/pip
+after_success:
+- codecov
 
-before_cache:
-  - rm -f $HOME/.cache/pip/log/debug.log
+_helpers:
+- &_mainstream_python_base
+  python: *mainstream_python
+- &_reset_steps
+  env: []
+  before_install: skip
+  install: skip
+  script: skip
+  after_success: skip
+- &_doc_base
+  stage: doc
+  <<: *_mainstream_python_base
+  <<: *_reset_steps
+  install:
+  - *upgrade_python_toolset
+  - pip install -U -r requirements/dev.txt -r requirements/doc.txt -r requirements/doc-spelling.txt
+  after_failure: cat docs/_build/spelling/output.txt
+  addons:
+    apt:
+      packages:
+      - libenchant-dev
 
-install:
-  - sudo apt-get install enchant
-  - pip install --upgrade pip wheel
-  - pip install --upgrade setuptools
-  - pip install --upgrade setuptools-git
-  - pip install -r requirements/ci.txt
-  - pip install aiodns
-  - pip install codecov
-  - if python -c "import sys; sys.exit(sys.version_info < (3,5))"; then
-        pip install uvloop;
-    fi
-  - pip install sphinxcontrib-spelling
+# doesn't work on MacOSX out of the box -- the system has no Python installed
+# there's a workaround to use `language: generic` and install it, but it's slow
+os: linux
 
-script:
-  - make cov-dev-full
+jobs:
+  fast_finish: true
+  allow_failures:
+  - python: 3.6-dev
+  - python: nightly
 
-  - if python -c "import sys; sys.exit(sys.version_info < (3,5))"; then
-        make doc;
-        make doc-spelling;
-    fi
+  include:
+  - python: 3.6-dev
+  - python: nightly
 
-after_success:
-  - codecov
-  - ./tools/run_docker.sh
+  - <<: *_doc_base
+    script:
+    - make doc-spelling
 
-deploy:
-  provider: pypi
-  user: andrew.svetlov
-  password:
-    secure: ZQKbdPT9BlNqP5CTbWRQyeyig7Bpf7wsnYVQIQPOZc9Ec74A+dsbagstR1sPkAO+d+5PN0pZMovvmU7OQhSVPAnJ74nsN90/fL4ux3kqYecMbevv0rJg20hMXSSkwMEIpjUsMdMjJvZAcaKytGWmKL0qAlOJHhixd1pBbWyuIUE=
-  distributions: "sdist"
-  on:
-    tags: true
-    all_branches: true
-    python: 3.6
+  - stage: deploy (PYPI upload itself runs only for tagged commits)
+    <<: *_mainstream_python_base
+    <<: *_reset_steps
+    dist: trusty
+    group: edge
+    services:
+    - docker
+    before_install:
+    # This must prevent further job progress
+    - |
+      if [ -z "$TRAVIS_TAG" ]
+      then
+        echo Not building wheels
+        exit 0
+      fi
+    script:
+    - ./tools/run_docker.sh "aiohttp"
+    deploy:
+      provider: pypi
+      # `skip_cleanup: true` is required to preserve binary wheels, built
+      # inside of manylinux1 docker container during `script` step above.
+      skip_cleanup: true
+      user: andrew.svetlov
+      password:
+        secure: ZQKbdPT9BlNqP5CTbWRQyeyig7Bpf7wsnYVQIQPOZc9Ec74A+dsbagstR1sPkAO+d+5PN0pZMovvmU7OQhSVPAnJ74nsN90/fL4ux3kqYecMbevv0rJg20hMXSSkwMEIpjUsMdMjJvZAcaKytGWmKL0qAlOJHhixd1pBbWyuIUE=
+      # Although Travis CI instructs `setup.py` to build source distribution,
+      # which is default value for distribution option (`distribution: sdist`),
+      # it will also upload all wheels we've previously built in manylinux1
+      # docker container using `twine upload -r pypi dist/*` command.
+      # Also since commit https://github.com/travis-ci/dpl/commit/90b5e39
+      # it is default that Travis PYPI provider has `skip_upload_docs: true`
+      # set by default.
+      # Besides above, we don't do cleanup of `dist/*`, because it's being done
+      # by Travis CI PYPI deployment provider after upload, unconditionally.
+      on:
+        tags: true
+        all_branches: true
+
+cache: pip
+
+before_cache:
+- rm -f $HOME/.cache/pip/log/debug.log

CONTRIBUTORS.txt

@@ -157,6 +157,7 @@ Stephen Granade
 Steven Seguin
 Sunghyun Hwang
 Sviatoslav Bulbakha
+Sviatoslav Sydorenko
 Taha Jahangir
 Taras Voinarovskyi
 Terence Honles

Makefile

@@ -52,13 +52,17 @@ cov-dev: .develop
 	@py.test --cov=aiohttp --cov-report=term --cov-report=html --cov-append tests
         @echo "open file://`pwd`/coverage/index.html"
 
-cov-dev-full: .develop
+cov-ci-no-ext: .develop
 	@echo "Run without extensions"
 	@AIOHTTP_NO_EXTENSIONS=1 py.test --cov=aiohttp tests
+cov-ci-aio-debug: .develop
 	@echo "Run in debug mode"
 	@PYTHONASYNCIODEBUG=1 py.test --cov=aiohttp --cov-append tests
+cov-ci-run: .develop
 	@echo "Regular run"
 	@py.test --cov=aiohttp --cov-report=term --cov-report=html --cov-append tests
+
+cov-dev-full: cov-ci-no-ext cov-ci-aio-debug cov-ci-run
 	@echo "open file://`pwd`/coverage/index.html"
 
 clean:

appveyor.yml

@@ -14,7 +14,6 @@ environment:
 
 install:
   - "tools/build.cmd %PYTHON%\\python.exe -m pip install wheel"
-  - "tools/build.cmd %PYTHON%\\python.exe -m pip install twine"
   - "tools/build.cmd %PYTHON%\\python.exe -m pip install -r requirements/ci.txt"
 
 build: false

requirements/ci.txt

@@ -1,3 +1,5 @@
+setuptools-git
+
 -r doc.txt
 pip==9.0.1
 flake8==3.3.0
@@ -14,7 +16,11 @@ pytest==3.1.3
 pytest-cov==2.5.1
 pytest-mock==1.6.0
 gunicorn==19.7.1
-#aiodns  # Enable from .travis.yml as required c-ares will not build on windows
 twine==1.9.1
 yarl==0.12.0
 -e .
+
+# Using PEP 508 env markers to control dependency on runtimes:
+aiodns==1.1.1; platform_system!="Windows"  # required c-ares will not build on windows
+codecov; platform_system!="Windows"  # We only use it in Travis CI
+uvloop; python_version>="3.5" and platform_system!="Windows"  # MagicStack/uvloop#14

requirements/dev.txt

@@ -2,5 +2,4 @@
 ipdb==0.10.3
 pytest-sugar==0.8.0
 ipython==6.1.0
-aiodns==1.1.1
 towncrier==17.4.0

requirements/doc-spelling.txt

@@ -0,0 +1 @@
+sphinxcontrib-spelling; platform_system!="Windows"  # We only use it in Travis CI

tools/build-wheels.sh

@@ -1,23 +1,47 @@
 #!/bin/bash
 PYTHON_VERSIONS="cp34-cp34m cp35-cp35m cp36-cp36m"
 
+# Avoid creation of __pycache__/*.py[c|o]
+export PYTHONDONTWRITEBYTECODE=1
+
+package_name="$1"
+if [ -z "$package_name" ]
+then
+    &>2 echo "Please pass package name as a first argument of this script ($0)"
+    exit 1
+fi
+
+arch=`uname -m`
+
+echo
+echo
 echo "Compile wheels"
 for PYTHON in ${PYTHON_VERSIONS}; do
     /opt/python/${PYTHON}/bin/pip install -r /io/requirements/wheel.txt
     /opt/python/${PYTHON}/bin/pip wheel /io/ -w /io/dist/
 done
 
+echo
+echo
 echo "Bundle external shared libraries into the wheels"
-for whl in /io/dist/aiohttp*.whl; do
-    auditwheel repair $whl -w /io/dist/
+for whl in /io/dist/${package_name}*${arch}.whl; do
+    echo "Repairing $whl..."
+    auditwheel repair "$whl" -w /io/dist/
 done
 
+echo "Cleanup OS specific wheels"
+rm -fv /io/dist/*-linux_*.whl
+
+echo
+echo
 echo "Install packages and test"
+echo "dist directory:"
+ls /io/dist
+
 for PYTHON in ${PYTHON_VERSIONS}; do
-    /opt/python/${PYTHON}/bin/pip install aiohttp --no-index -f file:///io/dist
-    rm -rf /io/tests/__pycache__
-    rm -rf /io/tests/test_py35/__pycache__
+    echo
+    echo -n "Test $PYTHON: "
+    /opt/python/${PYTHON}/bin/python -c "import platform;print(platform.platform())"
+    /opt/python/${PYTHON}/bin/pip install "$package_name" --no-index -f file:///io/dist
     /opt/python/${PYTHON}/bin/py.test /io/tests
-    rm -rf /io/tests/__pycache__
-    rm -rf /io/tests/test_py35/__pycache__
 done

tools/run_docker.sh

@@ -1,25 +1,32 @@
-if [ ! -z $TRAVIS_TAG ] && [ -z $PYTHONASYNCIODEBUG ] && [ -z $AIOHTTP_NO_EXTENSIONS] ;then
-    echo "x86_64"
-    docker pull quay.io/pypa/manylinux1_x86_64
-    docker run --rm -v `pwd`:/io quay.io/pypa/manylinux1_x86_64 /io/tools/build-wheels.sh
-    echo "Dist folder content is:"
-    for f in dist/aiohttp*manylinux1_x86_64.whl
-    do
-        echo "Upload $f"
-        python -m twine upload $f --username andrew.svetlov --password $PYPI_PASSWD
-    done
-    echo "Cleanup"
-    docker run --rm -v `pwd`:/io quay.io/pypa/manylinux1_x86_64 rm -rf /io/dist
-
-    echo "i686"
-    docker pull quay.io/pypa/manylinux1_i686
-    docker run --rm -v `pwd`:/io quay.io/pypa/manylinux1_i686 linux32 /io/tools/build-wheels.sh
-    echo "Dist folder content is:"
-    for f in dist/aiohttp*manylinux1_i686.whl
-    do
-        echo "Upload $f"
-        python -m twine upload $f --username andrew.svetlov --password $PYPI_PASSWD
-    done
-    echo "Cleanup"
-    docker run --rm -v `pwd`:/io quay.io/pypa/manylinux1_i686 rm -rf /io/dist
+#!/bin/bash
+package_name="$1"
+if [ -z "$package_name" ]
+then
+    &>2 echo "Please pass package name as a first argument of this script ($0)"
+    exit 1
 fi
+
+manylinux1_image_prefix="quay.io/pypa/manylinux1_"
+dock_ext_args=""
+declare -A docker_pull_pids=()  # This syntax requires at least bash v4
+
+for arch in x86_64 i686
+do
+    docker pull "${manylinux1_image_prefix}${arch}" &
+    docker_pull_pids[$arch]=$!
+done
+
+for arch in x86_64 i686
+do
+    echo
+    echo
+    arch_pull_pid=${docker_pull_pids[$arch]}
+    echo waiting for docker pull pid $arch_pull_pid to complete downloading container for $arch arch...
+    wait $arch_pull_pid  # await for docker image for current arch to be pulled from hub
+    [ $arch == "i686" ] && dock_ext_args="linux32"
+
+    echo Building wheel for $arch arch
+    docker run --rm -v `pwd`:/io "${manylinux1_image_prefix}${arch}" $dock_ext_args /io/tools/build-wheels.sh "$package_name"
+
+    dock_ext_args=""  # Reset docker args, just in case
+done