From afaf33a57a39b8b5b2087ea57dbad55597f4fb84 Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Sun, 4 Jun 2017 01:20:15 +0300 Subject: [PATCH 01/37] Use pipelines in Travis CI --- .travis.yml | 69 +++++++++++++++++++++++++++-------------------------- 1 file changed, 35 insertions(+), 34 deletions(-) diff --git a/.travis.yml b/.travis.yml index 4aba6a9a40d..d621887853a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,36 +4,48 @@ services: language: python -python: - # python3.4.2 has bug in http.cookies module, aiohttp provides fix - - 3.4.2 - - 3.4.3 - - 3.5.2 - # - 3.5.3 - - 3.5 - - 3.6 - # - 3.7-dev - # - 'nightly' +addons: + apt: + packages: + - enchant -os: - - linux -# - osx # doesn't work on MacOSX -- the system has no Python installed +# doesn't work on MacOSX out of the box -- the system has no Python installed +# there's a workaround to use `language: generic` and install it, but it's slow +os: linux -matrix: +jobs: allow_failures: - - python: '3.6-dev' - - python: 'nightly' - - os: osx + - python: 3.6-dev + - python: nightly -cache: - directories: - - $HOME/.cache/pip + include: + # python3.4.2 has bug in http.cookies module, aiohttp provides fix + - python: 3.4.2 + - python: 3.4.3 + - python: 3.5.2 + # - 3.5.3 + - python: 3.5 + - python: &mainstream_python 3.6 + - python: 3.6-dev + - python: nightly -before_cache: - - rm -f $HOME/.cache/pip/log/debug.log + - stage: deploy (PYPI upload itself runs only for tagged commits) + python: *mainstream_python + deploy: + provider: pypi + user: andrew.svetlov + password: + secure: ZQKbdPT9BlNqP5CTbWRQyeyig7Bpf7wsnYVQIQPOZc9Ec74A+dsbagstR1sPkAO+d+5PN0pZMovvmU7OQhSVPAnJ74nsN90/fL4ux3kqYecMbevv0rJg20hMXSSkwMEIpjUsMdMjJvZAcaKytGWmKL0qAlOJHhixd1pBbWyuIUE= + distributions: sdist + on: + tags: true + all_branches: true + +cache: pip + +before_cache: rm -f $HOME/.cache/pip/log/debug.log install: - - sudo apt-get install enchant - pip install --upgrade pip wheel - pip install --upgrade setuptools - pip install --upgrade setuptools-git @@ -56,14 +68,3 @@ script: after_success: - codecov - ./tools/run_docker.sh - -deploy: - provider: pypi - user: andrew.svetlov - password: - secure: ZQKbdPT9BlNqP5CTbWRQyeyig7Bpf7wsnYVQIQPOZc9Ec74A+dsbagstR1sPkAO+d+5PN0pZMovvmU7OQhSVPAnJ74nsN90/fL4ux3kqYecMbevv0rJg20hMXSSkwMEIpjUsMdMjJvZAcaKytGWmKL0qAlOJHhixd1pBbWyuIUE= - distributions: "sdist" - on: - tags: true - all_branches: true - python: 3.6 From 2f0343cb88ff6a1fac583c4345d4730db218350e Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Sun, 4 Jun 2017 01:22:52 +0300 Subject: [PATCH 02/37] Don't run test-related stages in deploy job Closes #1926 --- .travis.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.travis.yml b/.travis.yml index d621887853a..d0a4a6489e4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -31,6 +31,9 @@ jobs: - stage: deploy (PYPI upload itself runs only for tagged commits) python: *mainstream_python + install: skip + script: skip + after_success: skip deploy: provider: pypi user: andrew.svetlov From e2c7b56a1a202f1ffa79809915d43570ad61c5db Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Sun, 2 Jul 2017 21:19:58 +0300 Subject: [PATCH 03/37] Use environmental marker for uvloop install --- .travis.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index d0a4a6489e4..82032d71fab 100644 --- a/.travis.yml +++ b/.travis.yml @@ -55,9 +55,7 @@ install: - pip install -r requirements/ci.txt - pip install aiodns - pip install codecov - - if python -c "import sys; sys.exit(sys.version_info < (3,5))"; then - pip install uvloop; - fi + - pip install 'uvloop; python_version>="3.5"' - pip install sphinxcontrib-spelling script: From 6cfb00ac8a92011ee46f090222988b4d004a761e Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Sun, 2 Jul 2017 21:21:04 +0300 Subject: [PATCH 04/37] Optimize docker running script * Introduce package_name script param * Reduce nesting * Reduce code duplication --- .travis.yml | 2 +- tools/run_docker.sh | 46 ++++++++++++++++++++++++++------------------- 2 files changed, 28 insertions(+), 20 deletions(-) diff --git a/.travis.yml b/.travis.yml index 82032d71fab..33fd740f978 100644 --- a/.travis.yml +++ b/.travis.yml @@ -68,4 +68,4 @@ script: after_success: - codecov - - ./tools/run_docker.sh + - ./tools/run_docker.sh "aiohttp" diff --git a/tools/run_docker.sh b/tools/run_docker.sh index 056eb51d583..7d0bfbf2e36 100755 --- a/tools/run_docker.sh +++ b/tools/run_docker.sh @@ -1,25 +1,33 @@ -if [ ! -z $TRAVIS_TAG ] && [ -z $PYTHONASYNCIODEBUG ] && [ -z $AIOHTTP_NO_EXTENSIONS] ;then - echo "x86_64" - docker pull quay.io/pypa/manylinux1_x86_64 - docker run --rm -v `pwd`:/io quay.io/pypa/manylinux1_x86_64 /io/tools/build-wheels.sh - echo "Dist folder content is:" - for f in dist/aiohttp*manylinux1_x86_64.whl - do - echo "Upload $f" - python -m twine upload $f --username andrew.svetlov --password $PYPI_PASSWD - done - echo "Cleanup" - docker run --rm -v `pwd`:/io quay.io/pypa/manylinux1_x86_64 rm -rf /io/dist +#!/bin/bash +if [ -z $TRAVIS_TAG ] || [ ! -z $PYTHONASYNCIODEBUG ] || [ ! -z $AIOHTTP_NO_EXTENSIONS ] +then + exit 1 +fi + +package_name="$1" +if [ -z "$package_name" ] +then + &>2 echo "Please pass package name as a first argument of this script ($0)" + exit 1 +fi + +dock_ext_args="" + +for arch in x86_64 i686 +do + [ $arch == "i686" ] && dock_ext_args="linux32" + + echo "${arch}" + docker pull "quay.io/pypa/manylinux1_${arch}" + docker run --rm -v `pwd`:/io "quay.io/pypa/manylinux1_${arch}" $dock_ext_args /io/tools/build-wheels.sh "$package_name" - echo "i686" - docker pull quay.io/pypa/manylinux1_i686 - docker run --rm -v `pwd`:/io quay.io/pypa/manylinux1_i686 linux32 /io/tools/build-wheels.sh echo "Dist folder content is:" - for f in dist/aiohttp*manylinux1_i686.whl + for f in dist/aiohttp*manylinux1_${arch}.whl do echo "Upload $f" - python -m twine upload $f --username andrew.svetlov --password $PYPI_PASSWD + python -m twine upload "$f" --username andrew.svetlov --password "$PYPI_PASSWD" done + echo "Cleanup" - docker run --rm -v `pwd`:/io quay.io/pypa/manylinux1_i686 rm -rf /io/dist -fi + docker run --rm -v `pwd`:/io "quay.io/pypa/manylinux1_${arch}" rm -rf /io/dist +done From 06dadef4ffa6d0bab9890eb39a5e7a64ba6865c4 Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Sun, 2 Jul 2017 21:23:20 +0300 Subject: [PATCH 05/37] Introduce package_name param @ build wheels script --- tools/build-wheels.sh | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/tools/build-wheels.sh b/tools/build-wheels.sh index 4c2f2724daa..cff482c2321 100755 --- a/tools/build-wheels.sh +++ b/tools/build-wheels.sh @@ -1,6 +1,13 @@ #!/bin/bash PYTHON_VERSIONS="cp34-cp34m cp35-cp35m cp36-cp36m" +package_name="$1" +if [ -z "$package_name" ] +then + &>2 echo "Please pass package name as a first argument of this script ($0)" + exit 1 +fi + echo "Compile wheels" for PYTHON in ${PYTHON_VERSIONS}; do /opt/python/${PYTHON}/bin/pip install -r /io/requirements/wheel.txt @@ -8,13 +15,13 @@ for PYTHON in ${PYTHON_VERSIONS}; do done echo "Bundle external shared libraries into the wheels" -for whl in /io/dist/aiohttp*.whl; do - auditwheel repair $whl -w /io/dist/ +for whl in /io/dist/${package_name}*.whl; do + auditwheel repair "$whl" -w /io/dist/ done echo "Install packages and test" for PYTHON in ${PYTHON_VERSIONS}; do - /opt/python/${PYTHON}/bin/pip install aiohttp --no-index -f file:///io/dist + /opt/python/${PYTHON}/bin/pip install "$package_name" --no-index -f file:///io/dist rm -rf /io/tests/__pycache__ rm -rf /io/tests/test_py35/__pycache__ /opt/python/${PYTHON}/bin/py.test /io/tests From 6b2024413a5f861f08dd33a78bdc0fbd62361201 Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Sun, 2 Jul 2017 21:23:54 +0300 Subject: [PATCH 06/37] Avoid pycache with PYTHONDONTWRITEBYTECODE env var --- tools/build-wheels.sh | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tools/build-wheels.sh b/tools/build-wheels.sh index cff482c2321..e9a45065112 100755 --- a/tools/build-wheels.sh +++ b/tools/build-wheels.sh @@ -1,6 +1,9 @@ #!/bin/bash PYTHON_VERSIONS="cp34-cp34m cp35-cp35m cp36-cp36m" +# Avoid creation of __pycache__/*.py[c|o] +export PYTHONDONTWRITEBYTECODE=1 + package_name="$1" if [ -z "$package_name" ] then @@ -22,9 +25,5 @@ done echo "Install packages and test" for PYTHON in ${PYTHON_VERSIONS}; do /opt/python/${PYTHON}/bin/pip install "$package_name" --no-index -f file:///io/dist - rm -rf /io/tests/__pycache__ - rm -rf /io/tests/test_py35/__pycache__ /opt/python/${PYTHON}/bin/py.test /io/tests - rm -rf /io/tests/__pycache__ - rm -rf /io/tests/test_py35/__pycache__ done From 7ddc4eb83b83ceba8fc40b0b40d139c9749c1855 Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Sun, 2 Jul 2017 21:56:16 +0300 Subject: [PATCH 07/37] Fail fast in Travis CI --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 33fd740f978..e15fa53d5b2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,6 +14,7 @@ addons: os: linux jobs: + fast_finish: true allow_failures: - python: 3.6-dev - python: nightly From a94a1be9a72e3c2aa4443165a5b9ebdb52880558 Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Sun, 2 Jul 2017 22:13:56 +0300 Subject: [PATCH 08/37] Clean up dist folder in host system Motivation: eliminate container spawn overhead --- tools/run_docker.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/run_docker.sh b/tools/run_docker.sh index 7d0bfbf2e36..0fc3faa9fdf 100755 --- a/tools/run_docker.sh +++ b/tools/run_docker.sh @@ -29,5 +29,5 @@ do done echo "Cleanup" - docker run --rm -v `pwd`:/io "quay.io/pypa/manylinux1_${arch}" rm -rf /io/dist + rm -rf ./dist done From 05152fb93d5699919866d9d20179ea375b5a51a2 Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Sun, 2 Jul 2017 22:21:17 +0300 Subject: [PATCH 09/37] Use simplier check flags in bash script --- tools/run_docker.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/run_docker.sh b/tools/run_docker.sh index 0fc3faa9fdf..453c0c468af 100755 --- a/tools/run_docker.sh +++ b/tools/run_docker.sh @@ -1,5 +1,5 @@ #!/bin/bash -if [ -z $TRAVIS_TAG ] || [ ! -z $PYTHONASYNCIODEBUG ] || [ ! -z $AIOHTTP_NO_EXTENSIONS ] +if [ -z $TRAVIS_TAG ] || [ -n $PYTHONASYNCIODEBUG ] || [ -n $AIOHTTP_NO_EXTENSIONS ] then exit 1 fi From 08da39754b5e9b816f9cc3a86bcd542d7420058e Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Sun, 2 Jul 2017 22:46:00 +0300 Subject: [PATCH 10/37] Don't require VM in Travis CI --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index e15fa53d5b2..1410f97a167 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,4 +1,4 @@ -sudo: required +sudo: false services: - docker From 511663fac7505e292ff652962e68d4bd54e126e7 Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Mon, 3 Jul 2017 17:24:00 +0300 Subject: [PATCH 11/37] Move docker dependency to deploy job only --- .travis.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 1410f97a167..df9b21fce85 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,4 @@ sudo: false -services: - - docker language: python @@ -32,8 +30,11 @@ jobs: - stage: deploy (PYPI upload itself runs only for tagged commits) python: *mainstream_python + services: + - docker install: skip - script: skip + script: + - ./tools/run_docker.sh "aiohttp" after_success: skip deploy: provider: pypi @@ -69,4 +70,3 @@ script: after_success: - codecov - - ./tools/run_docker.sh "aiohttp" From a0db35a93b2190d3ce099293fa4b493371776384 Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Mon, 3 Jul 2017 17:26:01 +0300 Subject: [PATCH 12/37] Move mainstream python var to helpers --- .travis.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index df9b21fce85..aaaf62d36f1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,6 +2,10 @@ sudo: false language: python +_helpers: +- &mainstream_python 3.6 +- &_mainstream_python_base + python: *mainstream_python addons: apt: packages: @@ -24,7 +28,7 @@ jobs: - python: 3.5.2 # - 3.5.3 - python: 3.5 - - python: &mainstream_python 3.6 + - <<: *_mainstream_python_base - python: 3.6-dev - python: nightly From c938f4c2844015a72cee5ea1886399187a49c3d9 Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Mon, 3 Jul 2017 17:26:55 +0300 Subject: [PATCH 13/37] Move common requirements to requirements/ci.txt --- .travis.yml | 10 ++-------- appveyor.yml | 1 - requirements/ci.txt | 9 ++++++++- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.travis.yml b/.travis.yml index aaaf62d36f1..905edefcf90 100644 --- a/.travis.yml +++ b/.travis.yml @@ -55,14 +55,8 @@ cache: pip before_cache: rm -f $HOME/.cache/pip/log/debug.log install: - - pip install --upgrade pip wheel - - pip install --upgrade setuptools - - pip install --upgrade setuptools-git - - pip install -r requirements/ci.txt - - pip install aiodns - - pip install codecov - - pip install 'uvloop; python_version>="3.5"' - - pip install sphinxcontrib-spelling + - pip install --upgrade pip setuptools wheel + - pip install -U -r requirements/ci.txt script: - make cov-dev-full diff --git a/appveyor.yml b/appveyor.yml index f6931d03da4..9778a5097f7 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -14,7 +14,6 @@ environment: install: - "tools/build.cmd %PYTHON%\\python.exe -m pip install wheel" - - "tools/build.cmd %PYTHON%\\python.exe -m pip install twine" - "tools/build.cmd %PYTHON%\\python.exe -m pip install -r requirements/ci.txt" build: false diff --git a/requirements/ci.txt b/requirements/ci.txt index 56b71d113b7..0259fe916b5 100644 --- a/requirements/ci.txt +++ b/requirements/ci.txt @@ -1,3 +1,5 @@ +setuptools-git + -r doc.txt pip==9.0.1 flake8==3.3.0 @@ -14,7 +16,12 @@ pytest==3.1.3 pytest-cov==2.5.1 pytest-mock==1.6.0 gunicorn==19.7.1 -#aiodns # Enable from .travis.yml as required c-ares will not build on windows twine==1.9.1 yarl==0.12.0 -e . + +# Using PEP 508 env markers to control dependency on runtimes: +aiodns; platform_system!="Windows" # required c-ares will not build on windows +codecov; platform_system!="Windows" # We only use it in Travis CI +uvloop; python_version>="3.5" and platform_system!="Windows" # MagicStack/uvloop#14 +sphinxcontrib-spelling; platform_system!="Windows" # We only use it in Travis CI From cdeacae38aaf5e448739390a199631543c2dcfa5 Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Mon, 3 Jul 2017 17:27:53 +0300 Subject: [PATCH 14/37] Move doc to separate stages, install enchant there --- .travis.yml | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/.travis.yml b/.travis.yml index 905edefcf90..486314c08bd 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,10 +6,14 @@ _helpers: - &mainstream_python 3.6 - &_mainstream_python_base python: *mainstream_python -addons: - apt: - packages: - - enchant +- &_doc_base + stage: doc + <<: *_mainstream_python_base + after_failure: cat docs/_build/spelling/output.txt + addons: + apt: + packages: + - enchant # doesn't work on MacOSX out of the box -- the system has no Python installed # there's a workaround to use `language: generic` and install it, but it's slow @@ -32,6 +36,13 @@ jobs: - python: 3.6-dev - python: nightly + - <<: *_doc_base + script: + - make doc + - <<: *_doc_base + script: + - make doc-spelling + - stage: deploy (PYPI upload itself runs only for tagged commits) python: *mainstream_python services: @@ -61,10 +72,5 @@ install: script: - make cov-dev-full - - if python -c "import sys; sys.exit(sys.version_info < (3,5))"; then - make doc; - make doc-spelling; - fi - after_success: - codecov From 7b42a41c021e39b5029063a99eb937204b046165 Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Mon, 3 Jul 2017 17:35:49 +0300 Subject: [PATCH 15/37] Install spellcheck requirements only in doc stage --- .travis.yml | 23 +++++++++++++---------- requirements/ci.txt | 1 - requirements/doc-spelling.txt | 1 + 3 files changed, 14 insertions(+), 11 deletions(-) create mode 100644 requirements/doc-spelling.txt diff --git a/.travis.yml b/.travis.yml index 486314c08bd..fb23214a43a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,6 +2,16 @@ sudo: false language: python +install: +- &upgrade_python_toolset pip install --upgrade pip setuptools wheel +- pip install -U -r requirements/ci.txt + +script: +- make cov-dev-full + +after_success: +- codecov + _helpers: - &mainstream_python 3.6 - &_mainstream_python_base @@ -9,6 +19,9 @@ _helpers: - &_doc_base stage: doc <<: *_mainstream_python_base + install: + - *upgrade_python_toolset + - pip install -U -r requirements/doc.txt -r requirements/doc-spelling.txt after_failure: cat docs/_build/spelling/output.txt addons: apt: @@ -64,13 +77,3 @@ jobs: cache: pip before_cache: rm -f $HOME/.cache/pip/log/debug.log - -install: - - pip install --upgrade pip setuptools wheel - - pip install -U -r requirements/ci.txt - -script: - - make cov-dev-full - -after_success: - - codecov diff --git a/requirements/ci.txt b/requirements/ci.txt index 0259fe916b5..347a36665db 100644 --- a/requirements/ci.txt +++ b/requirements/ci.txt @@ -24,4 +24,3 @@ yarl==0.12.0 aiodns; platform_system!="Windows" # required c-ares will not build on windows codecov; platform_system!="Windows" # We only use it in Travis CI uvloop; python_version>="3.5" and platform_system!="Windows" # MagicStack/uvloop#14 -sphinxcontrib-spelling; platform_system!="Windows" # We only use it in Travis CI diff --git a/requirements/doc-spelling.txt b/requirements/doc-spelling.txt new file mode 100644 index 00000000000..934d9426f0a --- /dev/null +++ b/requirements/doc-spelling.txt @@ -0,0 +1 @@ +sphinxcontrib-spelling; platform_system!="Windows" # We only use it in Travis CI From 8a0eb5438cf0e916e8b930eac9a52f7171fe0b89 Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Mon, 3 Jul 2017 17:49:33 +0300 Subject: [PATCH 16/37] Move aiodns version lock to requirements/ci.txt --- requirements/ci.txt | 2 +- requirements/dev.txt | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/requirements/ci.txt b/requirements/ci.txt index 347a36665db..17c739b9bdf 100644 --- a/requirements/ci.txt +++ b/requirements/ci.txt @@ -21,6 +21,6 @@ yarl==0.12.0 -e . # Using PEP 508 env markers to control dependency on runtimes: -aiodns; platform_system!="Windows" # required c-ares will not build on windows +aiodns==1.1.1; platform_system!="Windows" # required c-ares will not build on windows codecov; platform_system!="Windows" # We only use it in Travis CI uvloop; python_version>="3.5" and platform_system!="Windows" # MagicStack/uvloop#14 diff --git a/requirements/dev.txt b/requirements/dev.txt index a92b636d73e..2db07a943dc 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -2,5 +2,4 @@ ipdb==0.10.3 pytest-sugar==0.8.0 ipython==6.1.0 -aiodns==1.1.1 towncrier==17.4.0 From 1c7b690dae27bc66d1ad845fcf4337def212eae4 Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Mon, 3 Jul 2017 18:18:03 +0300 Subject: [PATCH 17/37] Upgrade CI to Ubuntu Trusty --- .travis.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.travis.yml b/.travis.yml index fb23214a43a..ab16f6e0d80 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,5 +1,8 @@ sudo: false +dist: trusty +group: edge + language: python install: From db21ab4bcd6d215e1be265dcd1da7af707638807 Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Tue, 4 Jul 2017 00:25:45 +0300 Subject: [PATCH 18/37] Install dev deps in doc as well --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index ab16f6e0d80..30d129ea260 100644 --- a/.travis.yml +++ b/.travis.yml @@ -24,7 +24,7 @@ _helpers: <<: *_mainstream_python_base install: - *upgrade_python_toolset - - pip install -U -r requirements/doc.txt -r requirements/doc-spelling.txt + - pip install -U -r requirements/dev.txt -r requirements/doc.txt -r requirements/doc-spelling.txt after_failure: cat docs/_build/spelling/output.txt addons: apt: From 6714b769f233806db062df46a7017250fade96f5 Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Wed, 12 Jul 2017 19:26:38 +0300 Subject: [PATCH 19/37] Avoid building wheels when not releasing 'em --- .travis.yml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 30d129ea260..03043e898a5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -65,7 +65,15 @@ jobs: - docker install: skip script: - - ./tools/run_docker.sh "aiohttp" + - | + if [ -n "$TRAVIS_TAG" ] + then + echo Proceeding to build wheels + ./tools/run_docker.sh "aiohttp" + else + echo Not building wheels + exit 0 + fi after_success: skip deploy: provider: pypi From 0ad784c9af3108ad763fcab8e51f2cb396c6f0a7 Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Fri, 14 Jul 2017 02:52:22 +0300 Subject: [PATCH 20/37] Skip repo cleanup in deploy step in Travis CI --- .travis.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.travis.yml b/.travis.yml index 03043e898a5..a39ba32d581 100644 --- a/.travis.yml +++ b/.travis.yml @@ -77,9 +77,15 @@ jobs: after_success: skip deploy: provider: pypi + # `skip_cleanup: true` is required to preserve binary wheels, built + # inside of manylinux1 docker container during `script` step above + skip_cleanup: true user: andrew.svetlov password: secure: ZQKbdPT9BlNqP5CTbWRQyeyig7Bpf7wsnYVQIQPOZc9Ec74A+dsbagstR1sPkAO+d+5PN0pZMovvmU7OQhSVPAnJ74nsN90/fL4ux3kqYecMbevv0rJg20hMXSSkwMEIpjUsMdMjJvZAcaKytGWmKL0qAlOJHhixd1pBbWyuIUE= + # Although we instruct `setup.py` to build source distribution, it will + # also upload all wheels we've previously built in manylinux1 docker + # container. distributions: sdist on: tags: true From c9540514af6ee99d53c007c2487c2ce468559ba6 Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Fri, 14 Jul 2017 02:55:42 +0300 Subject: [PATCH 21/37] Move group/dist definition into deploy step --- .travis.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index a39ba32d581..0fe839a5e40 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,8 +1,5 @@ sudo: false -dist: trusty -group: edge - language: python install: @@ -61,6 +58,8 @@ jobs: - stage: deploy (PYPI upload itself runs only for tagged commits) python: *mainstream_python + dist: trusty + group: edge services: - docker install: skip From 226df615f903bbc60c15c88f400a56bf1fc10022 Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Fri, 14 Jul 2017 08:15:21 +0300 Subject: [PATCH 22/37] Don't proceed doing deploy job stuff if not tagged --- .travis.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.travis.yml b/.travis.yml index 0fe839a5e40..282ed7b1b31 100644 --- a/.travis.yml +++ b/.travis.yml @@ -62,6 +62,14 @@ jobs: group: edge services: - docker + before_install: + # This must prevent further job progress + - | + if [ -z "$TRAVIS_TAG" ] + then + echo Not building wheels "(before cache)" + exit 0 + fi install: skip script: - | From d09333a2ab5e926a43d80ebaa54b3d878e3e2ee1 Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Fri, 14 Jul 2017 10:56:03 +0300 Subject: [PATCH 23/37] Split CI test targets --- Makefile | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 1867d66fede..d8766fb088c 100644 --- a/Makefile +++ b/Makefile @@ -52,13 +52,17 @@ cov-dev: .develop @py.test --cov=aiohttp --cov-report=term --cov-report=html --cov-append tests @echo "open file://`pwd`/coverage/index.html" -cov-dev-full: .develop +cov-ci-no-ext: .develop @echo "Run without extensions" @AIOHTTP_NO_EXTENSIONS=1 py.test --cov=aiohttp tests +cov-ci-aio-debug: .develop @echo "Run in debug mode" @PYTHONASYNCIODEBUG=1 py.test --cov=aiohttp --cov-append tests +cov-ci-run: .develop @echo "Regular run" @py.test --cov=aiohttp --cov-report=term --cov-report=html --cov-append tests + +cov-dev-full: cov-ci-no-ext cov-ci-aio-debug cov-ci-run @echo "open file://`pwd`/coverage/index.html" clean: From f877e45e326777c095e48d3a9a55525ad761ddaa Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Fri, 14 Jul 2017 10:58:01 +0300 Subject: [PATCH 24/37] Split Travis CI script step into 3 commands --- .travis.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 282ed7b1b31..3a571b140fa 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,7 +7,9 @@ install: - pip install -U -r requirements/ci.txt script: -- make cov-dev-full +- make cov-ci-no-ext +- make cov-ci-aio-debug +- make cov-ci-run after_success: - codecov From eceeed629d17fd4f6176454428b3fac64608c3bd Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Fri, 14 Jul 2017 13:00:50 +0300 Subject: [PATCH 25/37] Optimize Travis CI config --- .travis.yml | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/.travis.yml b/.travis.yml index 3a571b140fa..ac46551c2b5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -69,20 +69,12 @@ jobs: - | if [ -z "$TRAVIS_TAG" ] then - echo Not building wheels "(before cache)" + echo Not building wheels exit 0 fi install: skip script: - - | - if [ -n "$TRAVIS_TAG" ] - then - echo Proceeding to build wheels - ./tools/run_docker.sh "aiohttp" - else - echo Not building wheels - exit 0 - fi + - ./tools/run_docker.sh "aiohttp" after_success: skip deploy: provider: pypi From 2bae3d3b73a82a5b626db5021fcd42a633479aa1 Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Sun, 16 Jul 2017 15:00:35 +0300 Subject: [PATCH 26/37] Fix valid manylinux1 wheels build script --- tools/build-wheels.sh | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/tools/build-wheels.sh b/tools/build-wheels.sh index e9a45065112..447a62514f1 100755 --- a/tools/build-wheels.sh +++ b/tools/build-wheels.sh @@ -11,19 +11,37 @@ then exit 1 fi +arch=`uname -m` + +echo +echo echo "Compile wheels" for PYTHON in ${PYTHON_VERSIONS}; do /opt/python/${PYTHON}/bin/pip install -r /io/requirements/wheel.txt /opt/python/${PYTHON}/bin/pip wheel /io/ -w /io/dist/ done +echo +echo echo "Bundle external shared libraries into the wheels" -for whl in /io/dist/${package_name}*.whl; do +for whl in /io/dist/${package_name}*${arch}.whl; do + echo "Repairing $whl..." auditwheel repair "$whl" -w /io/dist/ done +echo "Cleanup OS specific wheels" +rm -fv /io/dist/*-linux_*.whl + +echo +echo echo "Install packages and test" +echo "dist directory:" +ls /io/dist + for PYTHON in ${PYTHON_VERSIONS}; do + echo + echo -n "Test $PYTHON: " + /opt/python/${PYTHON}/bin/python -c "import platform;print(platform.platform())" /opt/python/${PYTHON}/bin/pip install "$package_name" --no-index -f file:///io/dist /opt/python/${PYTHON}/bin/py.test /io/tests done From 726548ca5e1d46f9a236da3d5af17b219f29ec67 Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Sun, 16 Jul 2017 15:01:44 +0300 Subject: [PATCH 27/37] Drop useless env check from docker script --- tools/run_docker.sh | 5 ----- 1 file changed, 5 deletions(-) diff --git a/tools/run_docker.sh b/tools/run_docker.sh index 453c0c468af..130a6790455 100755 --- a/tools/run_docker.sh +++ b/tools/run_docker.sh @@ -1,9 +1,4 @@ #!/bin/bash -if [ -z $TRAVIS_TAG ] || [ -n $PYTHONASYNCIODEBUG ] || [ -n $AIOHTTP_NO_EXTENSIONS ] -then - exit 1 -fi - package_name="$1" if [ -z "$package_name" ] then From ee2b244a8c7e62600d92cac23aa3f5b8d5f170bd Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Sun, 16 Jul 2017 15:02:01 +0300 Subject: [PATCH 28/37] Parallelize pulling manylinux1 docker images --- tools/run_docker.sh | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/tools/run_docker.sh b/tools/run_docker.sh index 130a6790455..1ea8c3a373a 100755 --- a/tools/run_docker.sh +++ b/tools/run_docker.sh @@ -6,23 +6,27 @@ then exit 1 fi +manylinux1_image_prefix="quay.io/pypa/manylinux1_" dock_ext_args="" +declare -A docker_pull_pids=() # This syntax requires at least bash v4 for arch in x86_64 i686 do - [ $arch == "i686" ] && dock_ext_args="linux32" + docker pull "${manylinux1_image_prefix}${arch}" & + docker_pull_pids[$arch]=$! +done - echo "${arch}" - docker pull "quay.io/pypa/manylinux1_${arch}" - docker run --rm -v `pwd`:/io "quay.io/pypa/manylinux1_${arch}" $dock_ext_args /io/tools/build-wheels.sh "$package_name" +for arch in x86_64 i686 +do + echo + echo + arch_pull_pid=${docker_pull_pids[$arch]} + echo waiting for docker pull pid $arch_pull_pid to complete downloading container for $arch arch... + wait $arch_pull_pid # await for docker image for current arch to be pulled from hub + [ $arch == "i686" ] && dock_ext_args="linux32" - echo "Dist folder content is:" - for f in dist/aiohttp*manylinux1_${arch}.whl - do - echo "Upload $f" - python -m twine upload "$f" --username andrew.svetlov --password "$PYPI_PASSWD" - done + echo Building wheel for $arch arch + docker run --rm -v `pwd`:/io "${manylinux1_image_prefix}${arch}" $dock_ext_args /io/tools/build-wheels.sh "$package_name" - echo "Cleanup" - rm -rf ./dist + dock_ext_args="" # Reset docker args, just in case done From 2904f9b64f37dbe7959775079491f07c13d35272 Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Sun, 16 Jul 2017 15:03:36 +0300 Subject: [PATCH 29/37] Add comments explaining Travis deploy section --- .travis.yml | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index ac46551c2b5..024ab68668b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -79,15 +79,18 @@ jobs: deploy: provider: pypi # `skip_cleanup: true` is required to preserve binary wheels, built - # inside of manylinux1 docker container during `script` step above + # inside of manylinux1 docker container during `script` step above. skip_cleanup: true user: andrew.svetlov password: secure: ZQKbdPT9BlNqP5CTbWRQyeyig7Bpf7wsnYVQIQPOZc9Ec74A+dsbagstR1sPkAO+d+5PN0pZMovvmU7OQhSVPAnJ74nsN90/fL4ux3kqYecMbevv0rJg20hMXSSkwMEIpjUsMdMjJvZAcaKytGWmKL0qAlOJHhixd1pBbWyuIUE= - # Although we instruct `setup.py` to build source distribution, it will - # also upload all wheels we've previously built in manylinux1 docker - # container. - distributions: sdist + # Although Travis CI instructs `setup.py` to build source distribution, + # which is default value for distribution option (`distribution: sdist`), + # it will also upload all wheels we've previously built in manylinux1 + # docker container using `twine upload -r pypi dist/*` command. + # Also since commit https://github.com/travis-ci/dpl/commit/90b5e39 + # it is default that Travis PYPI provider has `skip_upload_docs: true` + # set by default. on: tags: true all_branches: true From 21650f28f1b4ae63b20c7a93e38739804c3a208c Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Sun, 16 Jul 2017 15:09:57 +0300 Subject: [PATCH 30/37] Port setup checks from multidict repo --- .travis.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.travis.yml b/.travis.yml index 024ab68668b..5b1d6613daa 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,6 +10,10 @@ script: - make cov-ci-no-ext - make cov-ci-aio-debug - make cov-ci-run +- python setup.py check -rm +- if python -c "import sys; sys.exit(sys.version_info < (3,6))"; then + python setup.py check -s; + fi after_success: - codecov From cf0fe91fb6a4da01c9ce5cb3af9ffac91fc9761b Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Sun, 16 Jul 2017 15:10:49 +0300 Subject: [PATCH 31/37] Install just libenchant-dev for doc spellchecker --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 5b1d6613daa..75101e36deb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -32,7 +32,7 @@ _helpers: addons: apt: packages: - - enchant + - libenchant-dev # doesn't work on MacOSX out of the box -- the system has no Python installed # there's a workaround to use `language: generic` and install it, but it's slow From f03f1484f661b6905d87eaac2749eccd69c83d89 Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Sun, 16 Jul 2017 15:11:41 +0300 Subject: [PATCH 32/37] Drop unnecessary html doc build stage --- .travis.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 75101e36deb..8336e8e6da3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -55,9 +55,6 @@ jobs: - python: 3.6-dev - python: nightly - - <<: *_doc_base - script: - - make doc - <<: *_doc_base script: - make doc-spelling From 7002d9fb3a3763ea6a8a5410ae36b8152826476e Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Sun, 16 Jul 2017 15:12:35 +0300 Subject: [PATCH 33/37] Add comment about dist deletion in PYPI deployer --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index 8336e8e6da3..ea8104b7d2c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -92,6 +92,8 @@ jobs: # Also since commit https://github.com/travis-ci/dpl/commit/90b5e39 # it is default that Travis PYPI provider has `skip_upload_docs: true` # set by default. + # Besides above, we don't do cleanup of `dist/*`, because it's being done + # by Travis CI PYPI deployment provider after upload, unconditionally. on: tags: true all_branches: true From 11ab5b480da7a8a5960b50b54627d363f1ac9e28 Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Sun, 16 Jul 2017 15:13:20 +0300 Subject: [PATCH 34/37] Use sequence for before_cache step in YAML --- .travis.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index ea8104b7d2c..f6a20331370 100644 --- a/.travis.yml +++ b/.travis.yml @@ -100,4 +100,5 @@ jobs: cache: pip -before_cache: rm -f $HOME/.cache/pip/log/debug.log +before_cache: +- rm -f $HOME/.cache/pip/log/debug.log From 4cb2f8a4d98f57dbad9d4021e3cb129865f99b49 Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Sun, 16 Jul 2017 15:14:42 +0300 Subject: [PATCH 35/37] Add _reset_steps mixin to Travis config --- .travis.yml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index f6a20331370..cb971c9b24e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -22,9 +22,16 @@ _helpers: - &mainstream_python 3.6 - &_mainstream_python_base python: *mainstream_python +- &_reset_steps + env: [] + before_install: skip + install: skip + script: skip + after_success: skip - &_doc_base stage: doc <<: *_mainstream_python_base + <<: *_reset_steps install: - *upgrade_python_toolset - pip install -U -r requirements/dev.txt -r requirements/doc.txt -r requirements/doc-spelling.txt @@ -61,6 +68,7 @@ jobs: - stage: deploy (PYPI upload itself runs only for tagged commits) python: *mainstream_python + <<: *_reset_steps dist: trusty group: edge services: @@ -73,10 +81,8 @@ jobs: echo Not building wheels exit 0 fi - install: skip script: - ./tools/run_docker.sh "aiohttp" - after_success: skip deploy: provider: pypi # `skip_cleanup: true` is required to preserve binary wheels, built From d1af8f38277394c85cd3601dce5cd420adfb269a Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Sun, 16 Jul 2017 15:15:16 +0300 Subject: [PATCH 36/37] Move default python versions matrix to root level --- .travis.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.travis.yml b/.travis.yml index cb971c9b24e..fdb5c62f812 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,6 +2,14 @@ sudo: false language: python +python: +# python3.4.2 has bug in http.cookies module, aiohttp provides fix for it +- 3.4.2 +- 3.4.3 +- 3.5.2 +- 3.5 +- &mainstream_python 3.6 + install: - &upgrade_python_toolset pip install --upgrade pip setuptools wheel - pip install -U -r requirements/ci.txt @@ -19,7 +27,6 @@ after_success: - codecov _helpers: -- &mainstream_python 3.6 - &_mainstream_python_base python: *mainstream_python - &_reset_steps @@ -52,13 +59,6 @@ jobs: - python: nightly include: - # python3.4.2 has bug in http.cookies module, aiohttp provides fix - - python: 3.4.2 - - python: 3.4.3 - - python: 3.5.2 - # - 3.5.3 - - python: 3.5 - - <<: *_mainstream_python_base - python: 3.6-dev - python: nightly @@ -67,7 +67,7 @@ jobs: - make doc-spelling - stage: deploy (PYPI upload itself runs only for tagged commits) - python: *mainstream_python + <<: *_mainstream_python_base <<: *_reset_steps dist: trusty group: edge From 9948b87748d1871481d3c4e024f83a56143396db Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Sun, 16 Jul 2017 15:21:05 +0300 Subject: [PATCH 37/37] Add myself to contributors as advised in checklist --- CONTRIBUTORS.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/CONTRIBUTORS.txt b/CONTRIBUTORS.txt index d56ed9d096d..e49959e702a 100644 --- a/CONTRIBUTORS.txt +++ b/CONTRIBUTORS.txt @@ -157,6 +157,7 @@ Stephen Granade Steven Seguin Sunghyun Hwang Sviatoslav Bulbakha +Sviatoslav Sydorenko Taha Jahangir Taras Voinarovskyi Terence Honles