diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100644 index b7269b66ee..0000000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,98 +0,0 @@ -version: 2.1 - -orbs: - python: circleci/python@3.0.0 - -jobs: - manylinux_2_28-aarch64: - - parameters: - NRN_PYTHON_VERSION_MINOR: - type: string - NRN_NIGHTLY_UPLOAD: - type: string - - machine: - image: default - - resource_class: arm.medium - - steps: - - checkout - - run: - name: Build manylinux AArch64 wheel - command: | - docker run --rm \ - -w /root/nrn \ - -v $PWD:/root/nrn \ - -v /opt/nrnwheel/mpt:/nrnwheel/mpt \ - -e NEURON_NIGHTLY_TAG \ - -e NRN_NIGHTLY_UPLOAD \ - -e NRN_RELEASE_UPLOAD \ - -e SETUPTOOLS_SCM_PRETEND_VERSION \ - -e NRN_BUILD_FOR_UPLOAD=1 \ - 'docker.io/neuronsimulator/neuron_wheel:manylinux_2_28_aarch64' \ - packaging/python/build_wheels.bash linux 3<< parameters.NRN_PYTHON_VERSION_MINOR >> coreneuron - - - store_artifacts: - path: ./wheelhouse - destination: artifacts - - - run: - name: Test manylinux AArch64 wheel - command: | - sudo apt purge -y needrestart - sudo apt update - echo "deb http://ppa.launchpad.net/deadsnakes/ppa/ubuntu jammy main" | sudo tee /etc/apt/sources.list.d/deadsnakes-ppa.list && sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys F23C5A6CF475977595C89F51BA6932366A755776 && sudo apt update - sudo apt update - # install mpi dependencies - sudo apt install -y mpich openmpi-bin libopenmpi-dev libmpich-dev - version=3.<< parameters.NRN_PYTHON_VERSION_MINOR >> - # install Python from deadsnakes - sudo apt install -y python${version}-venv libpython${version}-dev g++ make - - export PYTHON_EXE=$(which python3.<< parameters.NRN_PYTHON_VERSION_MINOR >>) - - # test wheel - packaging/python/test_wheels.sh $PYTHON_EXE $(ls -t wheelhouse/*.whl) - - - run: - name: Upload nightly wheel to pypi.org - command: | - if [ "<< parameters.NRN_NIGHTLY_UPLOAD >>" == "true" ]; then - python -m pip install --upgrade pip - python -m pip install twine - python -m twine upload --verbose --skip-existing -u $TWINE_USERNAME -p $TWINE_PASSWORD wheelhouse/*.whl - else - echo "Skipping pypi.org upload!" - fi - -workflows: - - build-workflow: - jobs: - - manylinux_2_28-aarch64: - filters: - branches: - only: - - /release\/.*/ - - /circleci\/.*/ - matrix: - parameters: - NRN_PYTHON_VERSION_MINOR: ["13"] - NRN_NIGHTLY_UPLOAD: ["false"] - - nightly: - triggers: - - schedule: - cron: "0 0 * * *" - filters: - branches: - only: - - master - jobs: - - manylinux_2_28-aarch64: - matrix: - parameters: - NRN_PYTHON_VERSION_MINOR: ["9", "10", "11", "12", "13"] - NRN_NIGHTLY_UPLOAD: ["true"] diff --git a/.github/ISSUE_TEMPLATE/ci.md b/.github/ISSUE_TEMPLATE/ci.md index 33a66449cc..e90d4b445f 100644 --- a/.github/ISSUE_TEMPLATE/ci.md +++ b/.github/ISSUE_TEMPLATE/ci.md @@ -1,6 +1,6 @@ --- name: NEURON CI bug -about: Bugs related to the NEURON CIs (GitHub Actions, Azure, ReadTheDocs, CodeCov) +about: Bugs related to the NEURON CIs (GitHub Actions, ReadTheDocs, CodeCov) title: '' labels: 'bug, CI' assignees: '' diff --git a/.github/ISSUE_TEMPLATE/release-patch.md b/.github/ISSUE_TEMPLATE/release-patch.md index 32f822414d..61224b36e1 100644 --- a/.github/ISSUE_TEMPLATE/release-patch.md +++ b/.github/ISSUE_TEMPLATE/release-patch.md @@ -19,25 +19,24 @@ Pre-release - [ ] Update changelog below and agree on it with everyone; then commit it to `docs/changelog` in the cherrypicks PR (copy structure as-is) - [ ] Update `docs/index.rst` accordingly with the new `.pkg` and `.exe` links for `PKG installer` and `Windows Installer` - [ ] Activate ReadtheDocs for the cherry-pick branch and ensure the documentation builds (when logged in, go to [the versions page](https://readthedocs.org/projects/nrn/versions/) and set the version for your branch to Active and Hidden) -- [ ] Run a test wheel build WITHOUT upload on the cherry-pick branch to ensure all the wheels build ([see details](https://nrn.readthedocs.io/en/latest/install/python_wheels.html#publishing-the-wheels-on-pypi-via-azure)) +- [ ] Run a test wheel build WITHOUT upload on the cherry-pick branch to ensure all the wheels build ([see details](https://nrn.readthedocs.io/en/latest/install/python_wheels.html#publishing-the-wheels-on-pypi-via-github-actions)) Sanity checks --- -- [ ] After cherrypicks PR is merged, make sure GitHub, Azure and CircleCI builds pass for `release/x.y` branch -- [ ] Run [nrn-build-ci](https://github.com/neuronsimulator/nrn-build-ci/actions/workflows/build-neuron.yml) for the `release/x.y` branch; see [nrn-build-ci guide](https://github.com/neuronsimulator/nrn-build-ci#azure-wheels-testing---manual-workflow) +- [ ] After cherrypicks PR is merged, make sure GitHub Actions builds pass for `release/x.y` branch +- [ ] Run [nrn-build-ci](https://github.com/neuronsimulator/nrn-build-ci/actions/workflows/build-neuron.yml) for the `release/x.y` branch; see [nrn-build-ci guide](https://github.com/neuronsimulator/nrn-build-ci#wheels-testing---manual-workflow) - [ ] Activate ReadTheDocs build for `release/x.y` & make it hidden. Check docs are fine after build is done. -- [ ] Run BBP Simulation Stack & other relevant tests Releasing --- - [ ] Create new release+tag on GitHub via [release workflow](https://github.com/neuronsimulator/nrn/actions/workflows/release.yml?query=workflow%3A%22NEURON+Release%22). Note that the GitHub release will be marked as pre-release and will contain the full-src-package and the Windows installer at the end of the release workflow. -- [ ] Build release wheels but WITHOUT upload ([see details](https://nrn.readthedocs.io/en/latest/install/python_wheels.html#publishing-the-wheels-on-pypi-via-azure)) +- [ ] Build release wheels but WITHOUT upload ([see details](https://nrn.readthedocs.io/en/latest/install/python_wheels.html#publishing-the-wheels-on-pypi-via-github-actions)) - [ ] Create, test and upload manual artifacts - [ ] MacOS package installer (manual task, ask Michael) - [ ] arm64 wheels (manual task, check with Erik, Goran or Pramod) - [ ] aarch64 wheels (use existing `release/x.y-aarch64` branch for this, see [guide](https://nrn.readthedocs.io/en/latest/install/python_wheels.html#publishing-the-wheels-on-pypi-via-circleci)) -- [ ] Publish the `x.y.z` wheels on PyPI; see [wheel publishing instructions](https://nrn.readthedocs.io/en/latest/install/python_wheels.html#publishing-the-wheels-on-pypi-via-azure) +- [ ] Publish the `x.y.z` wheels on PyPI; see [wheel publishing instructions](https://nrn.readthedocs.io/en/latest/install/python_wheels.html#publishing-the-wheels-on-pypi-via-github-actions) - [ ] Once wheels are published, activate the `x.y.z` tag on ReadTheDocs - [ ] Rename the Windows installer in the GitHub release to match the new version and the supported python versions (i.e. `nrn-8.2.2.w64-mingw-py-39-310-311-312-setup.exe`) - [ ] Publish release on GitHub (edit https://github.com/neuronsimulator/nrn/releases/tag/x.y.z and un-tick the pre-release checkbox) diff --git a/.github/ISSUE_TEMPLATE/release.md b/.github/ISSUE_TEMPLATE/release.md index 7d2725184a..c5837f4bdb 100644 --- a/.github/ISSUE_TEMPLATE/release.md +++ b/.github/ISSUE_TEMPLATE/release.md @@ -16,11 +16,10 @@ Pre-release Sanity checks --- -- [ ] Create `release/x.y` branch and make sure GitHub, Azure and CircleCI builds pass -- [ ] Run [nrn-build-ci](https://github.com/neuronsimulator/nrn-build-ci/actions/workflows/build-neuron.yml) for the respective Azure build; see [Azure drop guide](https://github.com/neuronsimulator/nrn-build-ci#azure-wheels-testing---manual-workflow) +- [ ] Create `release/x.y` branch and make sure GitHub Actions builds pass +- [ ] Run [nrn-build-ci](https://github.com/neuronsimulator/nrn-build-ci/actions/workflows/build-neuron.yml) for the respective GitHub Actions build; see [the guide](https://github.com/neuronsimulator/nrn-build-ci#wheels-testing---manual-workflow) - [ ] Activate ReadTheDocs build for `release/x.y` & make it hidden. Check docs are fine after build is done. -- [ ] Run a test wheel build WITHOUT upload for `release/x.y` to ensure all the wheels build ([see details](https://nrn.readthedocs.io/en/latest/install/python_wheels.html#publishing-the-wheels-on-pypi-via-azure)) -- [ ] Run BBP Simulation Stack & other relevant tests +- [ ] Run a test wheel build WITHOUT upload for `release/x.y` to ensure all the wheels build ([see details](https://nrn.readthedocs.io/en/latest/install/python_wheels.html#publishing-the-wheels-on-pypi-via-github-actions)) Releasing @@ -30,12 +29,11 @@ Releasing - [ ] Update `docs/index.rst` accordingly with the new `.pkg` and `.exe` links for `PKG installer` and `Windows Installer` - [ ] Run the ReadTheDocs build again for `release-x.y`, make sure the build passes and inspect the Changelog page. - [ ] Create new release+tag on GitHub via [release workflow](https://github.com/neuronsimulator/nrn/actions/workflows/release.yml?query=workflow%3A%22NEURON+Release%22). Note that the GitHub release will be marked as pre-release and will contain the full-src-package and the Windows installer at the end of the release workflow. -- [ ] Build release wheels but WITHOUT upload ([see details](https://nrn.readthedocs.io/en/latest/install/python_wheels.html#publishing-the-wheels-on-pypi-via-azure)) +- [ ] Build release wheels but WITHOUT upload ([see details](https://nrn.readthedocs.io/en/latest/install/python_wheels.html#publishing-the-wheels-on-pypi-via-github-actions)) - [ ] Create, test and upload manual artifacts - [ ] MacOS package installer (manual task, ask Michael) - - [ ] arm64 wheels (manual task, check with Erik, Goran or Pramod) - [ ] aarch64 wheels (create a `release/x.y-aarch64` branch for this, see [guide](https://nrn.readthedocs.io/en/latest/install/python_wheels.html#publishing-the-wheels-on-pypi-via-circleci)) -- [ ] Publish the `x.y.z` wheels on Pypi; see [wheel publishing instructions](https://nrn.readthedocs.io/en/latest/install/python_wheels.html#publishing-the-wheels-on-pypi-via-azure) +- [ ] Publish the `x.y.z` wheels on Pypi; see [wheel publishing instructions](https://nrn.readthedocs.io/en/latest/install/python_wheels.html#publishing-the-wheels-on-pypi-via-github-actions) - [ ] Once wheels are published, activate the `x.y.z` tag on ReadTheDocs - [ ] Rename the Windows installer in the GitHub release to match the new version and the supported python versions (i.e. `nrn-8.2.2.w64-mingw-py-39-310-311-312-setup.exe` ) diff --git a/.github/workflows/external.yml b/.github/workflows/external.yml index ab06e55459..0150f2f9d7 100644 --- a/.github/workflows/external.yml +++ b/.github/workflows/external.yml @@ -13,26 +13,26 @@ env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} jobs: - get-last-azure-url: + get-last-artifact-url: runs-on: ubuntu-latest if: ${{ github.event.label.name == 'nrn-modeldb-ci-nightly' }} outputs: - azure_drop_url: ${{ steps.drop.outputs.azure_drop_url }} - pr_azure_sha: ${{ steps.drop.outputs.pr_azure_sha }} + artifact_url: ${{ steps.drop.outputs.artifact_url }} + last_commit_sha: ${{ steps.drop.outputs.last_commit_sha }} steps: - id: drop run: | - # use jq to get the last Azure drop URL from the PR and the SHA1 from the same body - export pr_json=$(gh pr view $PR_URL --json comments -q 'last(.comments[] .body | capture(".*(?[0-9a-f]{40}).*?(?https://dev.azure.com/neuronsimulator/.*=zip)"))') + # use jq to get the last artifact URL from the PR and the SHA1 from the same body + export pr_json=$(gh pr view $PR_URL --json comments -q 'last(.comments[] .body | capture(".*(?[0-9a-f]{40}).*?(?https://github.com/neuronsimulator/nrn/actions/runs/.*)"))') if [ -z "$pr_json" ]; then - echo "Unable to retrieve AZURE drop url and SHA from comments!" + echo "Unable to retrieve artifact URL and SHA from comments!" exit 1 fi - # use jq to get pr_azure_sha and azure_drop_url from the json - export azure_drop_url=$(echo $pr_json | jq -r .azure_drop_url) - export pr_azure_sha=$(echo $pr_json | jq -r .pr_azure_sha) - echo azure_drop_url=$azure_drop_url >> $GITHUB_OUTPUT - echo pr_azure_sha=$pr_azure_sha >> $GITHUB_OUTPUT + # use jq to get last_commit_sha and artifact_url from the json + export artifact_url=$(echo $pr_json | jq -r .artifact_url) + export last_commit_sha=$(echo $pr_json | jq -r .last_commit_sha) + echo artifact_url=$artifact_url >> $GITHUB_OUTPUT + echo last_commit_sha=$last_commit_sha >> $GITHUB_OUTPUT - id: remove-label if: always() @@ -41,31 +41,31 @@ jobs: gh pr edit $PR_URL --remove-label nrn-modeldb-ci-nightly # if we encounter an error in last github action step, add a comment if [ ${{ steps.drop.outcome }} == 'failure' ]; then - gh pr comment $PR_URL --body "Unable to retrieve AZURE drop url from comments!" + gh pr comment $PR_URL --body "Unable to retrieve artifact URL from comments!" else - gh pr comment $PR_URL --body "NEURON ModelDB CI: launching for ${pr_azure_sha} via its [drop url]($azure_drop_url)" + gh pr comment $PR_URL --body "NEURON ModelDB CI: launching for ${last_commit_sha} via its [artifact URL]($artifact_url)" fi env: - pr_azure_sha: ${{ steps.drop.outputs.pr_azure_sha }} - azure_drop_url: ${{ steps.drop.outputs.azure_drop_url }} + last_commit_sha: ${{ steps.drop.outputs.last_commit_sha }} + artifact_url: ${{ steps.drop.outputs.artifact_url }} nrn-modeldb-ci: - needs: get-last-azure-url + needs: get-last-artifact-url uses: neuronsimulator/nrn-modeldb-ci/.github/workflows/nrn-modeldb-ci.yaml@master with: - neuron_v1: ${{needs.get-last-azure-url.outputs.azure_drop_url}} + neuron_v1: ${{needs.get-last-artifact-url.outputs.artifact_url}} neuron_v2: neuron-nightly pr-update: needs: - nrn-modeldb-ci - - get-last-azure-url + - get-last-artifact-url runs-on: ubuntu-latest steps: - run: | - gh pr comment $PR_URL --body "NEURON ModelDB CI: ${pr_azure_sha} -> download reports [from here](${ARTIFACTS_URL})" + gh pr comment $PR_URL --body "NEURON ModelDB CI: ${last_commit_sha} -> download reports [from here](${ARTIFACTS_URL})" name: Post NEURON ModelDB CI Artifact URL if: always() || cancelled() env: ARTIFACTS_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} - pr_azure_sha: ${{ needs.get-last-azure-url.outputs.pr_azure_sha }} + last_commit_sha: ${{ needs.get-last-artifact-url.outputs.last_commit_sha }} diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml new file mode 100644 index 0000000000..4695e166c5 --- /dev/null +++ b/.github/workflows/wheels.yml @@ -0,0 +1,266 @@ +name: Build NEURON Python wheels + +on: + push: + branches: + - master + pull_request: + branches: + - master + schedule: + - cron: "0 0 * * *" + + workflow_dispatch: + inputs: + rel_release: + description: Release branch/commit + default: 'release/x.y' + required: true + rel_version: + description: Release version (tag name), in format major.minor.patch + default: '' + required: false + upload: + description: Whether to upload to PyPI or not + default: false + required: true + type: boolean + build_type: + description: The type of build (release or nightly) + default: 'nightly' + options: + - 'nightly' + - 'release' + base_image_x86_64: + description: The custom base container image to use for manylinux builds (x86_64) + base_image_aarch64: + description: The custom base container image to use for manylinux builds (aarch64) + +jobs: + build-test: + name: Build and test Python ${{ matrix.python-version }} wheel on ${{ matrix.os }} + runs-on: ${{ matrix.os }} + timeout-minutes: 60 + strategy: + matrix: + os: ['macos-13', 'macos-14', 'ubuntu-22.04', 'ubuntu-22.04-arm'] + python-version: ['3.9', '3.10', '3.11', '3.12', '3.13'] + include: + - python-version: '3.9' + python-org-version: '3.9.13' + python-installer-name: 'macos11.pkg' + + - python-version: '3.10' + python-org-version: '3.10.11' + python-installer-name: 'macos11.pkg' + + - python-version: '3.11' + python-org-version: '3.11.7' + python-installer-name: 'macos11.pkg' + + - python-version: '3.12' + python-org-version: '3.12.0' + python-installer-name: 'macos11.pkg' + + - python-version: '3.13' + python-org-version: '3.13.0' + python-installer-name: 'macos11.pkg' + + - os: 'ubuntu-22.04' + base_image: 'docker.io/neuronsimulator/neuron_wheel:manylinux_2_28_x86_64' + + - os: 'ubuntu-22.04-arm' + base_image: 'docker.io/neuronsimulator/neuron_wheel:manylinux_2_28_aarch64' + + steps: + - name: Check out code + if: github.event_name != 'workflow_dispatch' + uses: actions/checkout@v4 + with: + submodules: recursive + fetch-depth: 0 + + - name: Check out code for specific release + if: github.event_name == 'workflow_dispatch' + uses: actions/checkout@v4 + with: + ref: ${{ github.event.inputs.rel_release }} + submodules: recursive + fetch-depth: 0 + + - name: Install Python from python.org + if: runner.os == 'macOS' + run: | + installer="python-${{ matrix.python-org-version }}-${{ matrix.python-installer-name }}" + url="https://www.python.org/ftp/python/${{ matrix.python-org-version }}/${installer}" + curl $url -o $installer + sudo installer -pkg $installer -target / + + - name: Install System Dependencies + if: runner.os == 'macOS' + run: | + brew install --cask xquartz + brew install flex bison cmake mpich + brew unlink mpich && brew install openmpi + cmake --version + # Uninstall libomp for compatibility with issue #817 + brew uninstall --ignore-dependencies libomp || echo "libomp doesn't exist" + echo "$(brew --prefix)/opt/cmake/bin:$(brew --prefix)/opt/flex/bin:$(brew --prefix)/opt/bison/bin" >> $GITHUB_PATH + + - name: Install readline + if: runner.os == 'macOS' + run: | + sudo mkdir -p /opt/nrnwheel/$(uname -m) + sudo bash packaging/python/build_static_readline_osx.bash + + - name: Set env + run: | + BUILD_TYPE="${{ github.event.inputs.build_type }}" + UPLOAD="${{ github.event.inputs.upload }}" + # if they're empty, it means its not a workflow_dispatch, so we set a default + BUILD_TYPE="${BUILD_TYPE:-nightly}" + UPLOAD="${UPLOAD:-false}" + if [ "${BUILD_TYPE}" = 'nightly' ] + then + echo "NRN_NIGHTLY_UPLOAD=true" >> $GITHUB_ENV + echo "NEURON_NIGHTLY_TAG=-nightly" >> $GITHUB_ENV + else + echo "NRN_NIGHTLY_UPLOAD=false" >> $GITHUB_ENV + echo "NEURON_NIGHTLY_TAG=" >> $GITHUB_ENV + fi + + - name: Override env for scheduled run + if: github.event_name == 'schedule' + run: | + echo "NRN_NIGHTLY_UPLOAD=true" >> $GITHUB_ENV + echo "NEURON_NIGHTLY_TAG=-nightly" >> $GITHUB_ENV + + - name: Set custom version + if: github.event.inputs.rel_version != '' + run: | + echo SETUPTOOLS_SCM_PRETEND_VERSION=${{ github.event.inputs.rel_version }} >> $GITHUB_ENV + + - name: Build MacOS wheel + if: runner.os == 'macOS' + run: | + packaging/python/build_wheels.bash osx ${{ matrix.python-version }} coreneuron + + - name: Build Manylinux wheel + if: runner.os != 'macOS' + run: | + # manylinux does not support multi-platform images yet (see + # https://github.com/pypa/manylinux/issues/1306), so as a workaround, + # we use two custom input variables, one for x86, and one for arm + if [[ "${{ matrix.os }}" =~ .*arm$ ]] + then + BASE_IMAGE="${{ github.event.inputs.base_image_aarch64 }}" + BASE_IMAGE="${BASE_IMAGE:-${{ matrix.base_image }}}" + else + BASE_IMAGE="${{ github.event.inputs.base_image_x86_64 }}" + BASE_IMAGE="${BASE_IMAGE:-${{ matrix.base_image }}}" + fi + docker run --rm \ + -w /root/nrn \ + -v $(pwd):/root/nrn \ + -e NEURON_NIGHTLY_TAG \ + -e NRN_NIGHTLY_UPLOAD \ + -e SETUPTOOLS_SCM_PRETEND_VERSION \ + -e NRN_BUILD_FOR_UPLOAD=1 \ + ${BASE_IMAGE} \ + packaging/python/build_wheels.bash linux ${{ matrix.python-version }} coreneuron + + - name: Upload wheel files + uses: actions/upload-artifact@v4 + with: + name: wheels-${{ matrix.python-version }}-${{ matrix.os }} + path: wheelhouse/*.whl + + - name: Setup Python ${{ matrix.python-version }} for testing + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Install test dependencies + if: runner.os != 'macOS' + run: | + sudo apt update + sudo apt install -y mpich openmpi-bin libopenmpi-dev libmpich-dev + + - name: Test wheel with ${{ matrix.python-version }} + run: | + minor_version="$(python${{ matrix.python-version }} -c 'import sys;print(sys.version_info.minor)')" + packaging/python/test_wheels.sh $(which python${{ matrix.python-version }}) wheelhouse/*cp3${minor_version}*.whl + + merge: + name: Merge artifacts + runs-on: ubuntu-latest + needs: [build-test] + steps: + - name: Merge Artifacts + id: merge-artifacts + uses: actions/upload-artifact/merge@v4 + with: + delete-merged: true + name: wheels + pattern: wheels-* + + - name: Create comment with URL to artifact + if: github.event.pull_request + uses: peter-evans/create-or-update-comment@v4 + with: + issue-number: ${{ github.event.pull_request.number }} + body: | + ✔️ ${{ github.event.pull_request.head.sha }} -> [artifacts URL](${{ steps.merge-artifacts.outputs.artifact-url }}) + + pypi-publish-nightly: + name: Upload nightly wheels to PyPI + runs-on: ubuntu-latest + needs: merge + if: github.event_name == 'schedule' + environment: + name: pypi + url: https://pypi.org/p/neuron-nightly + permissions: + id-token: write # IMPORTANT: this permission is mandatory for trusted publishing + steps: + - name: Download wheels from artifact + uses: actions/download-artifact@v4 + with: + name: wheels + path: dist/ + - name: Publish package distributions to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + + pypi-publish-release: + name: Upload release wheels to PyPI + runs-on: ubuntu-latest + needs: merge + if: github.event_name == 'workflow_dispatch' && github.event.inputs.upload == 'true' + environment: + name: pypi + url: https://pypi.org/p/neuron + permissions: + id-token: write # IMPORTANT: this permission is mandatory for trusted publishing + steps: + - name: Download wheels from artifact + uses: actions/download-artifact@v4 + with: + name: wheels + path: dist/ + - name: Publish package distributions to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + + final: + name: Final CI + needs: [merge] + if: ${{ always() }} + runs-on: ubuntu-latest + steps: + - name: Check if CI matrix is successful + if: >- + ${{ + contains(needs.*.result, 'failure') + || contains(needs.*.result, 'cancelled') + || contains(needs.*.result, 'skipped') + }} + run: exit 1 diff --git a/azure-pipelines.yml b/azure-pipelines.yml deleted file mode 100644 index a081844ac0..0000000000 --- a/azure-pipelines.yml +++ /dev/null @@ -1,180 +0,0 @@ -#============================================================================= -# Azure Pipeline settings -#============================================================================= - -# Nightly build master for pypi upload -schedules: -- cron: "0 0 * * *" - branches: - include: - - master - always: true - -# Auto cancel old PR builds -pr: - autoCancel: true -# TODO : https://github.com/neuronsimulator/nrn/issues/1063 -# paths: -# exclude: -# - docs -# - README.md - -# Trigger build for certain branches only -trigger: -- master -- release/* - -stages: - - stage: BuildTestDeploy - jobs: - - - job: 'ManyLinuxWheels' - timeoutInMinutes: 45 - pool: - vmImage: 'ubuntu-20.04' - strategy: - matrix: - Python39: - python.version: '3.9' - Python310: - python.version: '3.10' - Python311: - python.version: '3.11' - Python312: - python.version: '3.12' - Python313: - python.version: '3.13' - - steps: - - # Secure files documentation: - # https://docs.microsoft.com/en-us/azure/devops/pipelines/library/secure-files?view=azure-devops - # NOTE: when uploading new secure files, access must be permitted from the Azure pipeline interface (check message there) - - task: DownloadSecureFile@1 - name: mpt_headersSF - displayName: 'Download mpt_headers secure file' - inputs: - secureFile: 'mpt_headears.2.21.tar.gz' - - # Note that mpt headers must be mounted in the docker imager under `/nrnwheel/mpt` - # This path is checked by `packaging/python/build_wheels.bash` when run in the image. - - script: | - sudo mkdir -p /opt/nrnwheel/mpt - sudo tar -zxf $(mpt_headersSF.secureFilePath) --directory /opt/nrnwheel/mpt - docker run --rm \ - -w /root/nrn \ - -v $PWD:/root/nrn \ - -v /opt/nrnwheel/mpt:/nrnwheel/mpt \ - -e NEURON_NIGHTLY_TAG \ - -e NRN_NIGHTLY_UPLOAD \ - -e NRN_RELEASE_UPLOAD \ - -e SETUPTOOLS_SCM_PRETEND_VERSION \ - -e NRN_BUILD_FOR_UPLOAD=1 \ - 'docker.io/neuronsimulator/neuron_wheel:manylinux_2_28_x86_64' \ - packaging/python/build_wheels.bash linux $(python.version) coreneuron - displayName: 'Building ManyLinux Wheel' - - - script: | - sudo apt update - sudo apt install -y mpich openmpi-bin libopenmpi-dev libmpich-dev - displayName: 'Install Test System Dependencies' - - - template: ci/azure-wheel-test-upload.yml - - - # Jobs to build OSX wheels natively - - job: 'MacOSWheels' - timeoutInMinutes: 60 - pool: - vmImage: 'macOS-13' - strategy: - matrix: - Python39: - python.version: '3.9' - python.org.version: '3.9.13' - python.installer.name: 'macos11.pkg' - Python310: - python.version: '3.10' - python.org.version: '3.10.11' - python.installer.name: 'macos11.pkg' - Python311: - python.version: '3.11' - python.org.version: '3.11.7' - python.installer.name: 'macos11.pkg' - Python312: - python.version: '3.12' - python.org.version: '3.12.0' - python.installer.name: 'macos11.pkg' - Python313: - python.version: '3.13' - python.org.version: '3.13.0' - python.installer.name: 'macos11.pkg' - - steps: - - - script: | - installer=python-$(python.org.version)-$(python.installer.name) - url=https://www.python.org/ftp/python/$(python.org.version)/$installer - curl $url -o $installer - sudo installer -pkg $installer -target / - displayName: 'Install Python from python.org' - - - script: | - brew install --cask xquartz - brew install flex bison mpich - brew unlink mpich && brew install openmpi - cmake --version - # see https://github.com/BlueBrain/CoreNeuron/issues/817, uninstall libomp until we fix this - # as we are building wheels, we shouldn't enable OpenMP here anyway - brew uninstall --ignore-dependencies libomp || echo "libomp doesn't exist" - displayName: 'Install OSX System Dependencies' - - # readline has been manually built with ncurses and MACOSX_DEPLOYMENT_TARGET=10.9 and stored as secure file on Azure. - # See `packaging/python/Dockerfile` for build instructions. - # - # Secure files documentation: - # https://docs.microsoft.com/en-us/azure/devops/pipelines/library/secure-files?view=azure-devops - # NOTE: when uploading new secure files, access must be permitted from the Azure pipeline interface (check message there) - - task: DownloadSecureFile@1 - name: readlineSF - displayName: 'Download readline secure file' - inputs: - secureFile: 'readline7.0-ncurses6.4.tar.gz' - - # 10.14 is required for full C++17 support according to - # https://cibuildwheel.readthedocs.io/en/stable/cpp_standards, but it - # seems that 10.15 is actually needed for std::filesystem::path. - - script: | - export MACOSX_DEPLOYMENT_TARGET=10.15 - export PATH=/usr/local/opt/flex/bin:/usr/local/opt/bison/bin:$PATH - export SDKROOT=$(xcrun --sdk macosx --show-sdk-path) - export NRN_BUILD_FOR_UPLOAD=1 - sudo mkdir -p /opt/nrnwheel/$(uname -m) - sudo tar -zxf $(readlineSF.secureFilePath) --directory /opt/nrnwheel/$(uname -m) - packaging/python/build_wheels.bash osx $(python.version) coreneuron - displayName: 'Build MacOS Wheel' - - - template: ci/azure-wheel-test-upload.yml - - - - stage: Final - jobs: - - job: AzureDropURL - pool: - vmImage: 'ubuntu-20.04' - condition: eq(variables['Build.Reason'], 'PullRequest') - steps: - - checkout: none - - script: | - export AZURE_DROP_URL=`curl -v 'https://dev.azure.com/neuronsimulator/nrn/_apis/build/builds/$(Build.BuildId)/artifacts?artifactName=drop' | jq -r '.resource.downloadUrl'` - echo "Setting dropurl to $AZURE_DROP_URL" - echo "##vso[task.setvariable variable=dropurl]$AZURE_DROP_URL" - displayName: 'Resolve Azure drop URL' - - - task: GitHubComment@0 - continueOnError: true - inputs: - gitHubConnection: 'neuronsimulator' - repositoryName: '$(Build.Repository.Name)' - comment: | - ✔️ $(system.pullRequest.sourceCommitId) -> [Azure artifacts URL]($(dropurl)) diff --git a/docs/install/python_wheels.md b/docs/install/python_wheels.md index 8456174bcd..491f9662c4 100644 --- a/docs/install/python_wheels.md +++ b/docs/install/python_wheels.md @@ -18,18 +18,18 @@ When required (i.e. update packages, add new software), `NEURON maintainers` are updating the NEURON docker images published on Docker Hub under [neuronsimulator/neuron_wheel](https://hub.docker.com/r/neuronsimulator/neuron_wheel). -Azure pipelines pull this image off DockerHub for Linux wheels building. +GitHub Actions pull this image off DockerHub for Linux wheels building. Updating and publishing the public images are done by a manual process that relies on a -`Docker file` (see [packaging/python/Dockerfile](../../packaging/python/Dockerfile)). -Any official update of these files shall imply a PR reviewed and merged before `DockerHub` publishing. +`Docker file` (see [packaging/python/Dockerfile](../../packaging/python/Dockerfile)). +Any official update of these files shall imply a PR reviewed and merged before publishing to `DockerHub`. -All wheels built on Azure are: +All wheels built on GitHub Actions are: * Published to `Pypi.org` as - * `neuron-nightly` -> when the pipeline is launched in CRON mode + * `neuron-nightly` -> when the pipeline is launched on a schedule * `neuron-x.y.z` -> when the pipeline is manually triggered for release `x.y.z` -* Stored as `Azure artifacts` in the Azure pipeline for every run. +* Stored as artifacts for every run. Refer to the following image for the NEURON Docker Image workflow: ![](images/docker-workflow.png) @@ -44,14 +44,14 @@ cd nrn/packaging/python docker build -t neuronsimulator/neuron_wheel: . ``` where `` is: -* `latest-x86_64` or `latest-aarch64` for official publishing on respective platforms. For `master`, we are using `latest-gcc9-x86_64` and `latest-gcc9-aarch64` (see [Use GCC9 for building wheels #1971](https://github.com/neuronsimulator/nrn/pull/1971)). -* `feature-name` for updates (for local testing or for PR testing purposes where you can temporarily publish the tag on DockerHub and tweak Azure CI pipelines to use it - refer to - `Job: 'ManyLinuxWheels'` in [azure-pipelines.yml](../../azure-pipelines.yml) ) +* `manylinux_2_28_x86_64` or `manylinux_2_28_aarch64` for official publishing on respective platforms. For `master`, we are using `manylinux_2_28_x86_64` and `manylinux_2_28_aarch64`. +* `feature-name` for updates (for local testing or for PR testing purposes where you can temporarily publish the tag on DockerHub and tweak GitHub Actions pipelines to use it - refer to + `Job: 'Build Manylinux wheel'` in [wheels.yml](../../.github/workflows/wheels.yml) ) -If you are building an image for AArch64 i.e. with `latest-aarch64` tag then you additionally pass `--build-arg` argument to docker build command in order to use compatible manylinux image for ARM64 platform (e.g. while building on Apple M1 or QEMU emulation): +If you are building an image for AArch64 i.e. with `manylinux_2_28_aarch64` tag then you additionally pass `--build-arg` argument to docker build command in order to use compatible manylinux image for ARM64 platform (e.g. while building on Apple M1 or QEMU emulation): ``` -docker build -t neuronsimulator/neuron_wheel:latest-aarch64 --build-arg MANYLINUX_IMAGE=manylinux2014_aarch64 -f Dockerfile . +docker build -t neuronsimulator/neuron_wheel:manylinux_2_28_aarch64 --build-arg MANYLINUX_IMAGE=manylinux_2_28_aarch64 -f Dockerfile . ``` @@ -60,25 +60,20 @@ docker build -t neuronsimulator/neuron_wheel:latest-aarch64 --build-arg MANYLINU In order to push the image and its tag: ``` docker login --username= -docker push neuronsimulator/neuron_wheel: +docker push docker.io/neuronsimulator/neuron_wheel: ``` ### Using the docker image -You can either build the neuron images locally or pull them from DockerHub: +You can either build the neuron images locally or pull them from DockerHub (below is the image for `x86_64`): ``` -$ docker pull neuronsimulator/neuron_wheel:latest-x86_64 -Using default tag: latest-x86_64 -latest: Pulling from neuronsimulator/neuron_wheel -.... -Status: Downloaded newer image for neuronsimulator/neuron_wheel:latest -docker.io/neuronsimulator/neuron_wheel:latest-x86_64 +docker pull docker.io/neuronsimulator/neuron_wheel:manylinux_2_28_x86_64 ``` We can conveniently mount the local NEURON repository inside docker, by using the `-v` option: ``` -docker run -v $PWD/nrn:/root/nrn -w /root/nrn -it neuronsimulator/neuron_wheel:latest-x86_64 bash +docker run -v $PWD/nrn:/root/nrn -w /root/nrn -it neuronsimulator/neuron_wheel:manylinux_2_28_x86_64 bash ``` where `$PWD/nrn` is a NEURON repository on the host machine that ends up mounted at `/root/nrn`. This is how you can test your NEURON updates inside the NEURON Docker image. @@ -90,7 +85,7 @@ The `neuronsimulator/neuron_wheel` provides out-of-the-box support for `mpich` a For `HPE-MPT MPI`, since it's not open source, you need to acquire the headers and mount them in the docker image: ``` -docker run -v $PWD/nrn:/root/nrn -w /root/nrn -v $PWD/mpt-headers/2.21/include:/nrnwheel/mpt/include -it neuronsimulator/neuron_wheel:latest-x86_64 bash +docker run -v $PWD/nrn:/root/nrn -w /root/nrn -v $PWD/mpt-headers/2.21/include:/nrnwheel/mpt/include -it neuronsimulator/neuron_wheel:manylinux_2_28_x86_64 bash ``` where `$PWD/mpt-headers` is the path to the HPE-MPT MPI headers on the host machine that end up mounted at `/nrnwheel/mpt/include`. @@ -99,19 +94,16 @@ where `$PWD/mpt-headers` is the path to the HPE-MPT MPI headers on the host mach Note that for macOS there is no docker image needed, but all required dependencies must exist. In order to have the wheels working on multiple macOS target versions, special consideration must be made for `MACOSX_DEPLOYMENT_TARGET`. - -Taking Azure macOS `x86_64` wheels for example, `readline` was built with `MACOSX_DEPLOYMENT_TARGET=10.9` and stored as secure file on Azure (under `Pipelines > Library > Secure files`). -For `arm64` we need to set `MACOSX_DEPLOYMENT_TARGET=11.0`. The wheels currently need to be built manually, using `universal2` Python installers. -For upcoming `universal2` wheels (targeting both `x86_64` and `arm64`) we will consider leveling everything to `MACOSX_DEPLOYMENT_TARGET=11.0`. - +Taking macOS `x86_64` wheels built by GitHub Actions for example, `readline` was built with `MACOSX_DEPLOYMENT_TARGET=10.15`. +For `arm64` we need to set `MACOSX_DEPLOYMENT_TARGET=11.0`. You can use [packaging/python/build_static_readline_osx.bash](../../packaging/python/build_static_readline_osx.bash) to build a static readline library. You can have a look at the script for requirements and usage. ### Installing macOS prerequisites -Install the necessary Python versions by downloading the universal2 installers from https://www.python.org/downloads/macos/ -You'll need several other packages installed as well (brew is fine): +Install the necessary Python versions by downloading the universal2 installers from [python.org](https://www.python.org/downloads/macos/). +You'll need several other packages installed as well (homebrew-built packages are fine): ``` brew install --cask xquartz @@ -120,11 +112,11 @@ brew unlink mpich && brew install openmpi brew uninstall --ignore-dependencies libomp || echo "libomp doesn't exist" ``` -Bison and flex installed through brew will not be symlinked into /opt/homebrew (installing it next to the version provided by OSX can cause problems). To ensure the installed versions will actually be picked up: +Note that bison and flex installed through brew will not be symlinked into `/opt/homebrew` or `/usr/local` (installing it next to the version provided by OSX can cause problems). To ensure the installed versions will actually be picked up by the NEURON installer: ``` -export BREW_PREFIX=$(brew --prefix) -export PATH=/opt/homebrew/opt/bison/bin:/opt/homebrew/opt/flex/bin:$PATH +export BREW_PREFIX="$(brew --prefix)" +export PATH="${BREW_PREFIX}/opt/bison/bin:${BREW_PREFIX}/opt/flex/bin:$PATH" ``` ## Launch the wheel building @@ -154,7 +146,7 @@ Where we are passing `3*` to build the wheels with `CoreNEURON` support for all You can also control the level of parallelization used for the build using the `NRN_PARALLEL_BUILDS` env variable (default: 4). ### macOS -As mentioned above, for macOS all dependencies have to be available on a system. You have to then clone NEURON repository and execute: +As mentioned above, for macOS all dependencies have to be available on a system. You have to then clone the NEURON repository and execute: ``` cd nrn @@ -186,106 +178,48 @@ bash packaging/python/test_wheels.sh python3.9 "-i https://test.pypi.org/simple/ On MacOS, launching `nrniv -python` or `special -python` can fail to load `neuron` module due to security restrictions. For this specific purpose, please `export SKIP_EMBEDED_PYTHON_TEST=true` before launching the tests. -## Publishing the wheels on Pypi via Azure +## Publishing the wheels on Pypi via GitHub Actions ### Variables that drive PyPI upload -We need to manipulate the following three predefined variables, listed hereafter with their default values: - * `NRN_NIGHTLY_UPLOAD` : `true` - * `NRN_RELEASE_UPLOAD` : `false` - * `NEURON_NIGHTLY_TAG` : `-nightly` +We need to manipulate the following two predefined variables, listed hereafter with their default values: + * `type` : `nightly` + * `upload` : `false` ### Release wheels -Head over to the [neuronsimulator.nrn](https://dev.azure.com/neuronsimulator/nrn/_build?definitionId=1) pipeline on Azure. +Head over to the [Build NEURON Python wheels](https://github.com/neuronsimulator/nrn/actions/workflows/wheels.yml) workflow on GitHub Actions. After creating the tag on the `release/x.y` or on the `master` branch, perform the following steps: -1) Click on `Run pipeline` -2) Input the release tag ref `refs/tags/x.y.z` -3) Click on `Advanced options` then select `Variables` -4) Update driving variables to: - * `NRN_NIGHTLY_UPLOAD` : `false` - * `NRN_RELEASE_UPLOAD` : `false` - * `NEURON_NIGHTLY_TAG` : undefined (leave empty) - - Do so by clicking `Variables` in `Advanced options` and update/clear the variable values. -5) Click on `Run` +1) Click on "Run workflow" +2) Input the release branch `release/x.y` in the field "Release branch/commit" +3) Update the following driving variables to: + * `type` : `release` + * `upload` : `false` +4) Click on the green `Run workflow` button. +TODO replace this image with one that uses GitHub Actions (or a generic artifacts cache). ![](images/azure-release-no-upload.png) -With above, wheel will be created like release from the provided tag but they won't be uploaded to the pypi.org ( as we have set `NRN_RELEASE_UPLOAD=false`). These wheels now you can download from artifacts section and perform thorough testing. Once you are happy with the testing result, set `NRN_RELEASE_UPLOAD` to `true` and trigger the pipeline same way: - * `NRN_NIGHTLY_UPLOAD` : `false` - * `NRN_RELEASE_UPLOAD` : `true` - * `NEURON_NIGHTLY_TAG` : undefined (leave empty) - - - -## Publishing the wheels on Pypi via CircleCI - -Currently CircleCI doesn't have automated pipeline for uploading `release` wheels to pypi.org (nightly wheels are uploaded automatically though). Currently we are using a **hacky**, semi-automated approach described below: - -* Checkout your tag as a new branch -* Update `.circleci/config.yml` as shown below -* Trigger CI pipeline manually for [the nrn project](https://app.circleci.com/pipelines/github/neuronsimulator/nrn) -* Upload wheels from artifacts manually - -``` -# checkout release tag as a new branch -$ git checkout 8.1a -b release/8.1a-aarch64 - -# manually updated `.circleci/config.yml` -$ git diff - -@@ -14,6 +14,11 @@ jobs: - - machine: - image: ubuntu-2004:202101-01 -+ environment: -+ SETUPTOOLS_SCM_PRETEND_VERSION: 8.2.6 -+ NEURON_NIGHTLY_TAG: "" -+ NRN_NIGHTLY_UPLOAD: false -+ NRN_RELEASE_UPLOAD: false - - resource_class: arm.medium - -@@ -54,6 +59,7 @@ jobs: - 39) pyenv_py_ver="3.9.1" ;; - 310) pyenv_py_ver="3.10.1" ;; - 311) pyenv_py_ver="3.11.0" ;; -+ 312) pyenv_py_ver="3.12.2" ;; - *) echo "Error: pyenv python version not specified!" && exit 1;; - esac - -@@ -95,7 +101,7 @@ workflows: - - /circleci\/.*/ - matrix: - parameters: -- NRN_PYTHON_VERSION: ["311"] -+ NRN_PYTHON_VERSION: ["39", "310", "311", "312"] - NRN_NIGHTLY_UPLOAD: ["false"] - - nightly: -``` - -The reason we are setting `SETUPTOOLS_SCM_PRETEND_VERSION` to a desired version `8.1a` because `setup.py` uses `git describe` and it will give different version name as we are now on a new branch! -`SETUPTOOLS_SCM_PRETEND_VERSION` will also stop your wheels from getting extra numbers on the version. - +With the above, wheels will be created like a release from the provided tag, but they won't be uploaded to PyPI (as we have set `upload=false`). These wheels can now be downloaded from the artifacts section and perform thorough testing. Once you are happy with the testing result, set `upload=true` and trigger the workflow the same way: + * `type` : `release` + * `upload` : `true` ## Nightly wheels -Nightly wheels get automatically published from `master` in CRON mode. - +Nightly wheels get automatically published on a schedule from the `master` branch. -## How to test Azure wheels locally +## How to test GitHub Actions wheels locally -After retrieving the Azure drop URL (i.e. from the GitHub PR comment, or by going to Azure for a specific build): +After download the wheel as an artifact, extract the archive to some directory: ```bash -python3 -m pip wheel neuron-gpu-nightly --wheel-dir tmp --find-links 'https://dev.azure.com/neuronsimulator/aa1fb98d-a914-45c3-a215-5e5ef1bd7687/_apis/build/builds/7600/artifacts?artifactName=drop&api-version=7.0&%24format=zip' +unzip -d tmp wheels.zip ``` -will download the wheel and its dependencies to `tmp/` and then you can test it with: + +You can then test a given wheel with: ```bash -./packaging/python/test_wheels.sh python3 ./tmp/NEURON_gpu_nightly-...whl true +./packaging/python/test_wheels.sh python3 ./tmp/NEURON_nightly-...whl true ``` diff --git a/packaging/python/build_wheels.bash b/packaging/python/build_wheels.bash index c022bdec7f..5c53f0d61b 100755 --- a/packaging/python/build_wheels.bash +++ b/packaging/python/build_wheels.bash @@ -30,8 +30,6 @@ py_ver="" python_requirements_path="$(mktemp -d)/requirements.txt" clone_nmodl_and_add_requirements() { - git config --global --add safe.directory /root/nrn - git submodule update --init --recursive --force --depth 1 -- external/nmodl # We only want the _build_ dependencies sed -e '/^# runtime dependencies/,$ d' external/nmodl/requirements.txt >> "${python_requirements_path}" }