diff --git a/.travis.yml b/.travis.yml index 4aba6a9a40d..fdb5c62f812 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,69 +1,110 @@ -sudo: required -services: - - docker +sudo: false language: python python: - # python3.4.2 has bug in http.cookies module, aiohttp provides fix - - 3.4.2 - - 3.4.3 - - 3.5.2 - # - 3.5.3 - - 3.5 - - 3.6 - # - 3.7-dev - # - 'nightly' +# python3.4.2 has bug in http.cookies module, aiohttp provides fix for it +- 3.4.2 +- 3.4.3 +- 3.5.2 +- 3.5 +- &mainstream_python 3.6 -os: - - linux -# - osx # doesn't work on MacOSX -- the system has no Python installed +install: +- &upgrade_python_toolset pip install --upgrade pip setuptools wheel +- pip install -U -r requirements/ci.txt -matrix: - allow_failures: - - python: '3.6-dev' - - python: 'nightly' - - os: osx +script: +- make cov-ci-no-ext +- make cov-ci-aio-debug +- make cov-ci-run +- python setup.py check -rm +- if python -c "import sys; sys.exit(sys.version_info < (3,6))"; then + python setup.py check -s; + fi -cache: - directories: - - $HOME/.cache/pip +after_success: +- codecov -before_cache: - - rm -f $HOME/.cache/pip/log/debug.log +_helpers: +- &_mainstream_python_base + python: *mainstream_python +- &_reset_steps + env: [] + before_install: skip + install: skip + script: skip + after_success: skip +- &_doc_base + stage: doc + <<: *_mainstream_python_base + <<: *_reset_steps + install: + - *upgrade_python_toolset + - pip install -U -r requirements/dev.txt -r requirements/doc.txt -r requirements/doc-spelling.txt + after_failure: cat docs/_build/spelling/output.txt + addons: + apt: + packages: + - libenchant-dev -install: - - sudo apt-get install enchant - - pip install --upgrade pip wheel - - pip install --upgrade setuptools - - pip install --upgrade setuptools-git - - pip install -r requirements/ci.txt - - pip install aiodns - - pip install codecov - - if python -c "import sys; sys.exit(sys.version_info < (3,5))"; then - pip install uvloop; - fi - - pip install sphinxcontrib-spelling +# doesn't work on MacOSX out of the box -- the system has no Python installed +# there's a workaround to use `language: generic` and install it, but it's slow +os: linux -script: - - make cov-dev-full +jobs: + fast_finish: true + allow_failures: + - python: 3.6-dev + - python: nightly - - if python -c "import sys; sys.exit(sys.version_info < (3,5))"; then - make doc; - make doc-spelling; - fi + include: + - python: 3.6-dev + - python: nightly -after_success: - - codecov - - ./tools/run_docker.sh + - <<: *_doc_base + script: + - make doc-spelling -deploy: - provider: pypi - user: andrew.svetlov - password: - secure: ZQKbdPT9BlNqP5CTbWRQyeyig7Bpf7wsnYVQIQPOZc9Ec74A+dsbagstR1sPkAO+d+5PN0pZMovvmU7OQhSVPAnJ74nsN90/fL4ux3kqYecMbevv0rJg20hMXSSkwMEIpjUsMdMjJvZAcaKytGWmKL0qAlOJHhixd1pBbWyuIUE= - distributions: "sdist" - on: - tags: true - all_branches: true - python: 3.6 + - stage: deploy (PYPI upload itself runs only for tagged commits) + <<: *_mainstream_python_base + <<: *_reset_steps + dist: trusty + group: edge + services: + - docker + before_install: + # This must prevent further job progress + - | + if [ -z "$TRAVIS_TAG" ] + then + echo Not building wheels + exit 0 + fi + script: + - ./tools/run_docker.sh "aiohttp" + deploy: + provider: pypi + # `skip_cleanup: true` is required to preserve binary wheels, built + # inside of manylinux1 docker container during `script` step above. + skip_cleanup: true + user: andrew.svetlov + password: + secure: ZQKbdPT9BlNqP5CTbWRQyeyig7Bpf7wsnYVQIQPOZc9Ec74A+dsbagstR1sPkAO+d+5PN0pZMovvmU7OQhSVPAnJ74nsN90/fL4ux3kqYecMbevv0rJg20hMXSSkwMEIpjUsMdMjJvZAcaKytGWmKL0qAlOJHhixd1pBbWyuIUE= + # Although Travis CI instructs `setup.py` to build source distribution, + # which is default value for distribution option (`distribution: sdist`), + # it will also upload all wheels we've previously built in manylinux1 + # docker container using `twine upload -r pypi dist/*` command. + # Also since commit https://github.com/travis-ci/dpl/commit/90b5e39 + # it is default that Travis PYPI provider has `skip_upload_docs: true` + # set by default. + # Besides above, we don't do cleanup of `dist/*`, because it's being done + # by Travis CI PYPI deployment provider after upload, unconditionally. + on: + tags: true + all_branches: true + +cache: pip + +before_cache: +- rm -f $HOME/.cache/pip/log/debug.log diff --git a/CONTRIBUTORS.txt b/CONTRIBUTORS.txt index d56ed9d096d..e49959e702a 100644 --- a/CONTRIBUTORS.txt +++ b/CONTRIBUTORS.txt @@ -157,6 +157,7 @@ Stephen Granade Steven Seguin Sunghyun Hwang Sviatoslav Bulbakha +Sviatoslav Sydorenko Taha Jahangir Taras Voinarovskyi Terence Honles diff --git a/Makefile b/Makefile index 1867d66fede..d8766fb088c 100644 --- a/Makefile +++ b/Makefile @@ -52,13 +52,17 @@ cov-dev: .develop @py.test --cov=aiohttp --cov-report=term --cov-report=html --cov-append tests @echo "open file://`pwd`/coverage/index.html" -cov-dev-full: .develop +cov-ci-no-ext: .develop @echo "Run without extensions" @AIOHTTP_NO_EXTENSIONS=1 py.test --cov=aiohttp tests +cov-ci-aio-debug: .develop @echo "Run in debug mode" @PYTHONASYNCIODEBUG=1 py.test --cov=aiohttp --cov-append tests +cov-ci-run: .develop @echo "Regular run" @py.test --cov=aiohttp --cov-report=term --cov-report=html --cov-append tests + +cov-dev-full: cov-ci-no-ext cov-ci-aio-debug cov-ci-run @echo "open file://`pwd`/coverage/index.html" clean: diff --git a/appveyor.yml b/appveyor.yml index f6931d03da4..9778a5097f7 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -14,7 +14,6 @@ environment: install: - "tools/build.cmd %PYTHON%\\python.exe -m pip install wheel" - - "tools/build.cmd %PYTHON%\\python.exe -m pip install twine" - "tools/build.cmd %PYTHON%\\python.exe -m pip install -r requirements/ci.txt" build: false diff --git a/requirements/ci.txt b/requirements/ci.txt index 56b71d113b7..17c739b9bdf 100644 --- a/requirements/ci.txt +++ b/requirements/ci.txt @@ -1,3 +1,5 @@ +setuptools-git + -r doc.txt pip==9.0.1 flake8==3.3.0 @@ -14,7 +16,11 @@ pytest==3.1.3 pytest-cov==2.5.1 pytest-mock==1.6.0 gunicorn==19.7.1 -#aiodns # Enable from .travis.yml as required c-ares will not build on windows twine==1.9.1 yarl==0.12.0 -e . + +# Using PEP 508 env markers to control dependency on runtimes: +aiodns==1.1.1; platform_system!="Windows" # required c-ares will not build on windows +codecov; platform_system!="Windows" # We only use it in Travis CI +uvloop; python_version>="3.5" and platform_system!="Windows" # MagicStack/uvloop#14 diff --git a/requirements/dev.txt b/requirements/dev.txt index a92b636d73e..2db07a943dc 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -2,5 +2,4 @@ ipdb==0.10.3 pytest-sugar==0.8.0 ipython==6.1.0 -aiodns==1.1.1 towncrier==17.4.0 diff --git a/requirements/doc-spelling.txt b/requirements/doc-spelling.txt new file mode 100644 index 00000000000..934d9426f0a --- /dev/null +++ b/requirements/doc-spelling.txt @@ -0,0 +1 @@ +sphinxcontrib-spelling; platform_system!="Windows" # We only use it in Travis CI diff --git a/tools/build-wheels.sh b/tools/build-wheels.sh index 4c2f2724daa..447a62514f1 100755 --- a/tools/build-wheels.sh +++ b/tools/build-wheels.sh @@ -1,23 +1,47 @@ #!/bin/bash PYTHON_VERSIONS="cp34-cp34m cp35-cp35m cp36-cp36m" +# Avoid creation of __pycache__/*.py[c|o] +export PYTHONDONTWRITEBYTECODE=1 + +package_name="$1" +if [ -z "$package_name" ] +then + &>2 echo "Please pass package name as a first argument of this script ($0)" + exit 1 +fi + +arch=`uname -m` + +echo +echo echo "Compile wheels" for PYTHON in ${PYTHON_VERSIONS}; do /opt/python/${PYTHON}/bin/pip install -r /io/requirements/wheel.txt /opt/python/${PYTHON}/bin/pip wheel /io/ -w /io/dist/ done +echo +echo echo "Bundle external shared libraries into the wheels" -for whl in /io/dist/aiohttp*.whl; do - auditwheel repair $whl -w /io/dist/ +for whl in /io/dist/${package_name}*${arch}.whl; do + echo "Repairing $whl..." + auditwheel repair "$whl" -w /io/dist/ done +echo "Cleanup OS specific wheels" +rm -fv /io/dist/*-linux_*.whl + +echo +echo echo "Install packages and test" +echo "dist directory:" +ls /io/dist + for PYTHON in ${PYTHON_VERSIONS}; do - /opt/python/${PYTHON}/bin/pip install aiohttp --no-index -f file:///io/dist - rm -rf /io/tests/__pycache__ - rm -rf /io/tests/test_py35/__pycache__ + echo + echo -n "Test $PYTHON: " + /opt/python/${PYTHON}/bin/python -c "import platform;print(platform.platform())" + /opt/python/${PYTHON}/bin/pip install "$package_name" --no-index -f file:///io/dist /opt/python/${PYTHON}/bin/py.test /io/tests - rm -rf /io/tests/__pycache__ - rm -rf /io/tests/test_py35/__pycache__ done diff --git a/tools/run_docker.sh b/tools/run_docker.sh index 056eb51d583..1ea8c3a373a 100755 --- a/tools/run_docker.sh +++ b/tools/run_docker.sh @@ -1,25 +1,32 @@ -if [ ! -z $TRAVIS_TAG ] && [ -z $PYTHONASYNCIODEBUG ] && [ -z $AIOHTTP_NO_EXTENSIONS] ;then - echo "x86_64" - docker pull quay.io/pypa/manylinux1_x86_64 - docker run --rm -v `pwd`:/io quay.io/pypa/manylinux1_x86_64 /io/tools/build-wheels.sh - echo "Dist folder content is:" - for f in dist/aiohttp*manylinux1_x86_64.whl - do - echo "Upload $f" - python -m twine upload $f --username andrew.svetlov --password $PYPI_PASSWD - done - echo "Cleanup" - docker run --rm -v `pwd`:/io quay.io/pypa/manylinux1_x86_64 rm -rf /io/dist - - echo "i686" - docker pull quay.io/pypa/manylinux1_i686 - docker run --rm -v `pwd`:/io quay.io/pypa/manylinux1_i686 linux32 /io/tools/build-wheels.sh - echo "Dist folder content is:" - for f in dist/aiohttp*manylinux1_i686.whl - do - echo "Upload $f" - python -m twine upload $f --username andrew.svetlov --password $PYPI_PASSWD - done - echo "Cleanup" - docker run --rm -v `pwd`:/io quay.io/pypa/manylinux1_i686 rm -rf /io/dist +#!/bin/bash +package_name="$1" +if [ -z "$package_name" ] +then + &>2 echo "Please pass package name as a first argument of this script ($0)" + exit 1 fi + +manylinux1_image_prefix="quay.io/pypa/manylinux1_" +dock_ext_args="" +declare -A docker_pull_pids=() # This syntax requires at least bash v4 + +for arch in x86_64 i686 +do + docker pull "${manylinux1_image_prefix}${arch}" & + docker_pull_pids[$arch]=$! +done + +for arch in x86_64 i686 +do + echo + echo + arch_pull_pid=${docker_pull_pids[$arch]} + echo waiting for docker pull pid $arch_pull_pid to complete downloading container for $arch arch... + wait $arch_pull_pid # await for docker image for current arch to be pulled from hub + [ $arch == "i686" ] && dock_ext_args="linux32" + + echo Building wheel for $arch arch + docker run --rm -v `pwd`:/io "${manylinux1_image_prefix}${arch}" $dock_ext_args /io/tools/build-wheels.sh "$package_name" + + dock_ext_args="" # Reset docker args, just in case +done