Skip to content

Commit 6b51bcd

Browse files
lguohanlguohan
authored
[ci]: add vstest to azure pipeline (sonic-net#1436)
Signed-off-by: Guohan Lu <[email protected]>
1 parent 28b64ec commit 6b51bcd

5 files changed

+314
-61
lines changed

‎.azure-pipelines/build-docker-sonic-vs-template.yml

+55
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
parameters:
2+
- name: timeout
3+
type: number
4+
default: 60
5+
6+
- name: artifact_name
7+
type: string
8+
9+
jobs:
10+
- job:
11+
displayName: "docker-sonic-vs"
12+
timeoutInMinutes: ${{ parameters.timeout }}
13+
14+
pool:
15+
vmImage: 'ubuntu-20.04'
16+
17+
steps:
18+
- task: DownloadPipelineArtifact@2
19+
inputs:
20+
artifact: wheels
21+
displayName: "Download sonic swss artifact"
22+
23+
- task: DownloadPipelineArtifact@2
24+
inputs:
25+
source: specific
26+
project: build
27+
pipeline: 1
28+
artifact: sonic-buildimage.vs
29+
runVersion: 'latestFromBranch'
30+
runBranch: 'refs/heads/master'
31+
displayName: "Download sonic buildimage"
32+
33+
- script: |
34+
set -ex
35+
36+
echo $(Build.DefinitionName).$(Build.BuildNumber)
37+
38+
docker load < ../target/docker-sonic-vs.gz
39+
40+
mkdir -p .azure-pipelines/docker-sonic-vs/wheels
41+
42+
cp -v ../*.whl .azure-pipelines/docker-sonic-vs/wheels
43+
44+
pushd .azure-pipelines
45+
46+
docker build --no-cache -t docker-sonic-vs:$(Build.DefinitionName).$(Build.BuildNumber) docker-sonic-vs
47+
48+
popd
49+
50+
docker save docker-sonic-vs:$(Build.DefinitionName).$(Build.BuildNumber) | gzip -c > $(Build.ArtifactStagingDirectory)/docker-sonic-vs.gz
51+
displayName: "Build docker-sonic-vs image"
52+
53+
- publish: $(Build.ArtifactStagingDirectory)/
54+
artifact: ${{ parameters.artifact_name }}
55+
displayName: "Archive sonic docker vs image"

‎.azure-pipelines/build_and_install_module.sh

+87
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,87 @@
1+
#!/bin/bash
2+
#
3+
# build and install team/vrf driver
4+
#
5+
6+
set -e
7+
8+
source /etc/os-release
9+
10+
function build_and_install_kmodule()
11+
{
12+
if sudo modprobe team 2>/dev/null && sudo modprobe vrf 2>/dev/null && sudo modprobe macsec 2>/dev/null; then
13+
echo "The module team, vrf and macsec exist."
14+
return
15+
fi
16+
17+
[ -z "$WORKDIR" ] && WORKDIR=$(mktemp -d)
18+
cd $WORKDIR
19+
20+
KERNEL_RELEASE=$(uname -r)
21+
KERNEL_MAINVERSION=$(echo $KERNEL_RELEASE | cut -d- -f1)
22+
EXTRAVERSION=$(echo $KERNEL_RELEASE | cut -d- -f2)
23+
LOCALVERSION=$(echo $KERNEL_RELEASE | cut -d- -f3)
24+
VERSION=$(echo $KERNEL_MAINVERSION | cut -d. -f1)
25+
PATCHLEVEL=$(echo $KERNEL_MAINVERSION | cut -d. -f2)
26+
SUBLEVEL=$(echo $KERNEL_MAINVERSION | cut -d. -f3)
27+
28+
# Install the required debian packages to build the kernel modules
29+
apt-get install -y build-essential linux-headers-${KERNEL_RELEASE} autoconf pkg-config fakeroot
30+
apt-get install -y flex bison libssl-dev libelf-dev
31+
apt-get install -y libnl-route-3-200 libnl-route-3-dev libnl-cli-3-200 libnl-cli-3-dev libnl-3-dev
32+
33+
# Add the apt source mirrors and download the linux image source code
34+
cp /etc/apt/sources.list /etc/apt/sources.list.bk
35+
sed -i "s/^# deb-src/deb-src/g" /etc/apt/sources.list
36+
apt-get update
37+
apt-get source linux-image-unsigned-$(uname -r) > source.log
38+
39+
# Recover the original apt sources list
40+
cp /etc/apt/sources.list.bk /etc/apt/sources.list
41+
apt-get update
42+
43+
# Build the Linux kernel module drivers/net/team and vrf
44+
cd $(find . -maxdepth 1 -type d | grep -v "^.$")
45+
make allmodconfig
46+
mv .config .config.bk
47+
cp /boot/config-$(uname -r) .config
48+
grep NET_TEAM .config.bk >> .config
49+
echo CONFIG_NET_VRF=m >> .config
50+
echo CONFIG_MACSEC=m >> .config
51+
make VERSION=$VERSION PATCHLEVEL=$PATCHLEVEL SUBLEVEL=$SUBLEVEL EXTRAVERSION=-${EXTRAVERSION} LOCALVERSION=-${LOCALVERSION} modules_prepare
52+
make M=drivers/net/team
53+
mv drivers/net/Makefile drivers/net/Makefile.bak
54+
echo 'obj-$(CONFIG_NET_VRF) += vrf.o' > drivers/net/Makefile
55+
echo 'obj-$(CONFIG_MACSEC) += macsec.o' >> drivers/net/Makefile
56+
make M=drivers/net
57+
58+
# Install the module
59+
TEAM_DIR=$(echo /lib/modules/$(uname -r)/kernel/net/team)
60+
NET_DIR=$(echo /lib/modules/$(uname -r)/kernel/net)
61+
if [ ! -e "$TEAM_DIR/team.ko" ]; then
62+
mkdir -p $TEAM_DIR
63+
cp drivers/net/team/*.ko $TEAM_DIR/
64+
modinfo $TEAM_DIR/team.ko
65+
depmod
66+
modprobe team
67+
fi
68+
if [ ! -e "$NET_DIR/vrf.ko" ]; then
69+
mkdir -p $NET_DIR
70+
cp drivers/net/vrf.ko $NET_DIR/
71+
modinfo $NET_DIR/vrf.ko
72+
depmod
73+
modprobe vrf
74+
fi
75+
if [ ! -e "$NET_DIR/macsec.ko" ]; then
76+
mkdir -p $NET_DIR
77+
cp drivers/net/macsec.ko $NET_DIR/
78+
modinfo $NET_DIR/macsec.ko
79+
depmod
80+
modprobe macsec
81+
fi
82+
83+
cd /tmp
84+
rm -rf $WORKDIR
85+
}
86+
87+
build_and_install_kmodule

‎.azure-pipelines/docker-sonic-vs/Dockerfile

+7
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
FROM docker-sonic-vs
2+
3+
ARG docker_container_name
4+
5+
ADD ["wheels", "/wheels"]
6+
7+
RUN pip3 install --no-deps --force-reinstall /wheels/sonic_utilities-1.2-py3-none-any.whl

‎.azure-pipelines/test-docker-sonic-vs-template.yml

+75
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
parameters:
2+
- name: timeout
3+
type: number
4+
default: 180
5+
6+
- name: log_artifact_name
7+
type: string
8+
9+
jobs:
10+
- job:
11+
displayName: vstest
12+
timeoutInMinutes: ${{ parameters.timeout }}
13+
14+
pool:
15+
vmImage: 'ubuntu-20.04'
16+
17+
steps:
18+
- task: DownloadPipelineArtifact@2
19+
inputs:
20+
artifact: docker-sonic-vs
21+
displayName: "Download docker sonic vs image"
22+
23+
- task: DownloadPipelineArtifact@2
24+
inputs:
25+
source: specific
26+
project: build
27+
pipeline: 9
28+
artifact: sonic-swss-common.amd64.ubuntu20_04
29+
runVersion: 'latestFromBranch'
30+
runBranch: 'refs/heads/master'
31+
displayName: "Download sonic swss common deb packages"
32+
33+
- checkout: self
34+
displayName: "Checkout sonic-swss-common"
35+
- checkout: sonic-swss
36+
displayName: "Checkout sonic-swss"
37+
38+
- script: |
39+
set -x
40+
sudo sonic-utilities/.azure-pipelines/build_and_install_module.sh
41+
42+
sudo apt-get install -y libhiredis0.14
43+
sudo dpkg -i --force-confask,confnew ../libswsscommon_1.0.0_amd64.deb || apt-get install -f
44+
sudo dpkg -i ../python3-swsscommon_1.0.0_amd64.deb
45+
46+
# install packages for vs test
47+
sudo apt-get install -y net-tools bridge-utils vlan
48+
sudo apt-get install -y python3-pip
49+
sudo pip3 install pytest==4.6.2 attrs==19.1.0 exabgp==4.0.10 distro==1.5.0 docker==4.4.1 redis==3.3.4 flaky==3.7.0
50+
displayName: "Install dependencies"
51+
52+
- script: |
53+
set -x
54+
sudo docker load -i ../docker-sonic-vs.gz
55+
docker ps
56+
ip netns list
57+
pushd sonic-swss/tests
58+
sudo py.test -v --force-flaky --junitxml=tr.xml --imgname=docker-sonic-vs:$(Build.DefinitionName).$(Build.BuildNumber)
59+
displayName: "Run vs tests"
60+
61+
- task: PublishTestResults@2
62+
inputs:
63+
testResultsFiles: '**/tr.xml'
64+
testRunTitle: vstest
65+
condition: always()
66+
67+
- script: |
68+
cp -r sonic-swss/tests/log $(Build.ArtifactStagingDirectory)/
69+
displayName: "Collect logs"
70+
condition: always()
71+
72+
- publish: $(Build.ArtifactStagingDirectory)/
73+
artifact: ${{ parameters.log_artifact_name }}@$(System.JobAttempt)
74+
displayName: "Publish logs"
75+
condition: always()

‎azure-pipelines.yml

+90-61
Original file line numberDiff line numberDiff line change
@@ -6,73 +6,102 @@
66
trigger:
77
- master
88

9-
pool:
10-
vmImage: ubuntu-20.04
9+
resources:
10+
repositories:
11+
- repository: sonic-swss
12+
type: github
13+
name: Azure/sonic-swss
14+
endpoint: build
1115

12-
container:
13-
image: sonicdev-microsoft.azurecr.io:443/sonic-slave-buster:latest
16+
stages:
17+
- stage: Build
1418

15-
steps:
16-
- task: DownloadPipelineArtifact@2
17-
inputs:
18-
source: specific
19-
project: build
20-
pipeline: 1
21-
artifact: sonic-buildimage.vs
22-
runVersion: 'latestFromBranch'
23-
runBranch: 'refs/heads/master'
24-
displayName: "Download artifacts from latest sonic-buildimage build"
19+
jobs:
20+
- job:
21+
displayName: "Python3"
22+
pool:
23+
vmImage: ubuntu-20.04
2524

26-
- script: |
27-
set -xe
28-
sudo dpkg -i libnl-3-200_*.deb
29-
sudo dpkg -i libnl-genl-3-200_*.deb
30-
sudo dpkg -i libnl-route-3-200_*.deb
31-
sudo dpkg -i libnl-nf-3-200_*.deb
32-
sudo dpkg -i libhiredis0.14_*.deb
33-
sudo dpkg -i libswsscommon_1.0.0_amd64.deb
34-
sudo dpkg -i python3-swsscommon_1.0.0_amd64.deb
35-
sudo dpkg -i libyang_1.0.73_amd64.deb
36-
sudo dpkg -i libyang-cpp_1.0.73_amd64.deb
37-
sudo dpkg -i python3-yang_1.0.73_amd64.deb
38-
workingDirectory: $(Pipeline.Workspace)/target/debs/buster/
39-
displayName: 'Install Debian dependencies'
25+
container:
26+
image: sonicdev-microsoft.azurecr.io:443/sonic-slave-buster:latest
4027

41-
- script: |
42-
set -xe
43-
sudo pip3 install swsssdk-2.0.1-py3-none-any.whl
44-
sudo pip3 install sonic_py_common-1.0-py3-none-any.whl
45-
sudo pip3 install sonic_config_engine-1.0-py3-none-any.whl
46-
sudo pip3 install sonic_platform_common-1.0-py3-none-any.whl
47-
sudo pip3 install sonic_yang_mgmt-1.0-py3-none-any.whl
48-
sudo pip3 install sonic_yang_models-1.0-py3-none-any.whl
49-
workingDirectory: $(Pipeline.Workspace)/target/python-wheels/
50-
displayName: 'Install Python dependencies'
28+
steps:
29+
- task: DownloadPipelineArtifact@2
30+
inputs:
31+
source: specific
32+
project: build
33+
pipeline: 1
34+
artifact: sonic-buildimage.vs
35+
runVersion: 'latestFromBranch'
36+
runBranch: 'refs/heads/master'
37+
displayName: "Download artifacts from latest sonic-buildimage build"
5138

52-
- script: |
53-
python3 setup.py test
54-
displayName: 'Test Python 3'
39+
- script: |
40+
set -xe
41+
sudo dpkg -i libnl-3-200_*.deb
42+
sudo dpkg -i libnl-genl-3-200_*.deb
43+
sudo dpkg -i libnl-route-3-200_*.deb
44+
sudo dpkg -i libnl-nf-3-200_*.deb
45+
sudo dpkg -i libhiredis0.14_*.deb
46+
sudo dpkg -i libswsscommon_1.0.0_amd64.deb
47+
sudo dpkg -i python3-swsscommon_1.0.0_amd64.deb
48+
sudo dpkg -i libyang_1.0.73_amd64.deb
49+
sudo dpkg -i libyang-cpp_1.0.73_amd64.deb
50+
sudo dpkg -i python3-yang_1.0.73_amd64.deb
51+
workingDirectory: $(Pipeline.Workspace)/target/debs/buster/
52+
displayName: 'Install Debian dependencies'
5553
56-
- task: PublishTestResults@2
57-
inputs:
58-
testResultsFiles: '$(System.DefaultWorkingDirectory)/test-results.xml'
59-
testRunTitle: Python 3
60-
failTaskOnFailedTests: true
61-
condition: succeededOrFailed()
62-
displayName: 'Publish Python 3 test results'
54+
- script: |
55+
set -xe
56+
sudo pip3 install swsssdk-2.0.1-py3-none-any.whl
57+
sudo pip3 install sonic_py_common-1.0-py3-none-any.whl
58+
sudo pip3 install sonic_config_engine-1.0-py3-none-any.whl
59+
sudo pip3 install sonic_platform_common-1.0-py3-none-any.whl
60+
sudo pip3 install sonic_yang_mgmt-1.0-py3-none-any.whl
61+
sudo pip3 install sonic_yang_models-1.0-py3-none-any.whl
62+
workingDirectory: $(Pipeline.Workspace)/target/python-wheels/
63+
displayName: 'Install Python dependencies'
6364
64-
- task: PublishCodeCoverageResults@1
65-
inputs:
66-
codeCoverageTool: Cobertura
67-
summaryFileLocation: '$(System.DefaultWorkingDirectory)/coverage.xml'
68-
reportDirectory: '$(System.DefaultWorkingDirectory)/htmlcov/'
69-
displayName: 'Publish Python 3 test coverage'
65+
- script: |
66+
python3 setup.py test
67+
displayName: 'Test Python 3'
7068
71-
- script: |
72-
set -e
73-
python3 setup.py bdist_wheel
74-
displayName: 'Build Python 3 wheel'
69+
- task: PublishTestResults@2
70+
inputs:
71+
testResultsFiles: '$(System.DefaultWorkingDirectory)/test-results.xml'
72+
testRunTitle: Python 3
73+
failTaskOnFailedTests: true
74+
condition: succeededOrFailed()
75+
displayName: 'Publish Python 3 test results'
7576

76-
- publish: '$(System.DefaultWorkingDirectory)/dist/'
77-
artifact: wheels
78-
displayName: "Publish Python wheels"
77+
- task: PublishCodeCoverageResults@1
78+
inputs:
79+
codeCoverageTool: Cobertura
80+
summaryFileLocation: '$(System.DefaultWorkingDirectory)/coverage.xml'
81+
reportDirectory: '$(System.DefaultWorkingDirectory)/htmlcov/'
82+
displayName: 'Publish Python 3 test coverage'
83+
84+
- script: |
85+
set -e
86+
python3 setup.py bdist_wheel
87+
displayName: 'Build Python 3 wheel'
88+
89+
- publish: '$(System.DefaultWorkingDirectory)/dist/'
90+
artifact: wheels
91+
displayName: "Publish Python wheels"
92+
93+
- stage: BuildDocker
94+
dependsOn: Build
95+
condition: succeeded('Build')
96+
jobs:
97+
- template: .azure-pipelines/build-docker-sonic-vs-template.yml
98+
parameters:
99+
artifact_name: docker-sonic-vs
100+
101+
- stage: Test
102+
dependsOn: BuildDocker
103+
condition: succeeded('BuildDocker')
104+
jobs:
105+
- template: .azure-pipelines/test-docker-sonic-vs-template.yml
106+
parameters:
107+
log_artifact_name: log

0 commit comments

Comments
 (0)