diff --git a/jenkins/common/sonic-swss-common-build-pr/Jenkinsfile b/jenkins/common/sonic-swss-common-build-pr/Jenkinsfile index 07e5f4c80a..fd83b7d9b2 100644 --- a/jenkins/common/sonic-swss-common-build-pr/Jenkinsfile +++ b/jenkins/common/sonic-swss-common-build-pr/Jenkinsfile @@ -1,5 +1,9 @@ pipeline { - agent { node { label 'sonic-slave-buster' } } + agent { node { label 'jenkins-vstest-workers' } } + + options { + buildDiscarder(logRotator(artifactDaysToKeepStr: '', artifactNumToKeepStr: '', daysToKeepStr: '', numToKeepStr: '30')) + } stages { stage('Prepare') { @@ -10,7 +14,27 @@ pipeline { userRemoteConfigs: [[url: 'http://github.com/Azure/sonic-swss-common', refspec: '+refs/pull/*:refs/remotes/origin/pr/*']]]) } - copyArtifacts(projectName: 'vs/buildimage-vs-all', filter: '**/*.deb', target: 'buildimage', flatten: false) + + dir('sairedis') { + checkout([$class: 'GitSCM', + branches: [[name: '*/master']], + extensions: [[$class: 'SubmoduleOption', + disableSubmodules: false, + parentCredentials: false, + recursiveSubmodules: true, + reference: '', + trackingSubmodules: false]], + userRemoteConfigs: [[url: 'http://github.com/Azure/sonic-sairedis']]]) + + } + + dir('swss') { + checkout([$class: 'GitSCM', + branches: [[name: '*/master']], + userRemoteConfigs: [[url: 'https://github.com/Azure/sonic-swss']]]) + } + + copyArtifacts(projectName: 'vs/buildimage-vs-all', filter: '**/*.deb,**/docker-sonic-vs.gz', target: 'buildimage', flatten: false) } } @@ -20,16 +44,28 @@ pipeline { } } - stage('Test') { + stage('swsscommon Unit Tests') { steps { sh './scripts/common/sonic-swss-common-build/test.sh' } } + + stage('Virtual Switch Tests') { + steps { + sh './scripts/vs/sonic-swss-build/test.sh' + } + } } + post { + always { + archiveArtifacts(artifacts: 'target/*swsscommon*.deb, swss/tests/tr.xml') + junit(allowEmptyResults: true, keepLongStdio: true, testResults: 'swss/tests/tr.xml') + cleanWs(disableDeferredWipeout: false, deleteDirs: true, notFailBuild: true) + } - success { - archiveArtifacts(artifacts: 'target/*.deb') + failure { + archiveArtifacts(artifacts: 'buildimage/target/docker-sonic-vs.gz, **/*.log, target/var/log/*, swss/tests/log/**') } } } diff --git a/jenkins/common/sonic-swss-common-build/Jenkinsfile b/jenkins/common/sonic-swss-common-build/Jenkinsfile index 6b9dc43d85..e2bbfd98eb 100644 --- a/jenkins/common/sonic-swss-common-build/Jenkinsfile +++ b/jenkins/common/sonic-swss-common-build/Jenkinsfile @@ -1,16 +1,12 @@ pipeline { - agent { node { label 'sonic-slave-buster' } } + agent { node { label 'jenkins-vstest-workers' } } options { buildDiscarder(logRotator(artifactDaysToKeepStr: '', artifactNumToKeepStr: '', daysToKeepStr: '', numToKeepStr: '10')) } - environment { - SONIC_TEAM_WEBHOOK = credentials('public-jenkins-builder') - } - triggers { - pollSCM('H/10 * * * *') + cron('H H/12 * * *') } stages { @@ -21,7 +17,27 @@ pipeline { branches: [[name: 'refs/heads/master']], userRemoteConfigs: [[url: 'http://github.com/Azure/sonic-swss-common']]]) } - copyArtifacts(projectName: '../vs/buildimage-vs-all', filter: '**/*.deb', target: 'buildimage', flatten: false) + + dir('sairedis') { + checkout([$class: 'GitSCM', + branches: [[name: '*/master']], + extensions: [[$class: 'SubmoduleOption', + disableSubmodules: false, + parentCredentials: false, + recursiveSubmodules: true, + reference: '', + trackingSubmodules: false]], + userRemoteConfigs: [[url: 'http://github.com/Azure/sonic-sairedis']]]) + + } + + dir('swss') { + checkout([$class: 'GitSCM', + branches: [[name: '*/master']], + userRemoteConfigs: [[url: 'https://github.com/Azure/sonic-swss']]]) + } + + copyArtifacts(projectName: 'vs/buildimage-vs-all', filter: '**/*.deb,**/docker-sonic-vs.gz', target: 'buildimage', flatten: false) } } @@ -31,23 +47,30 @@ pipeline { } } - stage('Test') { + stage('swsscommon Unit Tests') { steps { sh './scripts/common/sonic-swss-common-build/test.sh' } } + + stage('Virtual Switch Tests') { + steps { + catchError(buildResult: 'SUCCESS', stageResult: 'UNSTABLE') { + sh './scripts/vs/sonic-swss-build/test.sh' + } + } + } } + post { - success { - archiveArtifacts(artifacts: 'target/*.deb') + always { + archiveArtifacts(artifacts: 'target/*swsscommon*.deb, swss/tests/tr.xml') + junit(allowEmptyResults: true, keepLongStdio: true, testResults: 'swss/tests/tr.xml') + cleanWs(disableDeferredWipeout: false, deleteDirs: true, notFailBuild: true) } - fixed { - slackSend(color:'#00FF00', message: "Build job back to normal: ${env.JOB_NAME} ${env.BUILD_NUMBER} (<${env.BUILD_URL}|Open>)") - office365ConnectorSend(webhookUrl: "${env.SONIC_TEAM_WEBHOOK}") - } - regression { - slackSend(color:'#FF0000', message: "Build job Regression: ${env.JOB_NAME} ${env.BUILD_NUMBER} (<${env.BUILD_URL}|Open>)") - office365ConnectorSend(webhookUrl: "${env.SONIC_TEAM_WEBHOOK}") + + failure { + archiveArtifacts(artifacts: 'buildimage/target/docker-sonic-vs.gz, **/*.log, target/var/log/*, swss/tests/log/**') } } } diff --git a/scripts/common/sonic-swss-common-build/build.sh b/scripts/common/sonic-swss-common-build/build.sh index aac16c314a..66480240e5 100755 --- a/scripts/common/sonic-swss-common-build/build.sh +++ b/scripts/common/sonic-swss-common-build/build.sh @@ -1,30 +1,17 @@ #!/bin/bash -ex -# Install swig -sudo apt-get install -y swig +echo ${JOB_NAME##*/}.${BUILD_NUMBER} -# Install HIREDIS -sudo apt-get install -y libhiredis0.14 libhiredis-dev +docker pull sonicdev-microsoft.azurecr.io:443/sonic-slave-buster-johnar:latest +docker run --rm=true --privileged -v $(pwd):/sonic -w /sonic -i sonicdev-microsoft.azurecr.io:443/sonic-slave-buster-johnar:latest ./scripts/common/sonic-swss-common-build/docker_build_script.sh -# Install libnl3 -sudo dpkg -i buildimage/target/debs/buster/libnl-3-200_*.deb -sudo dpkg -i buildimage/target/debs/buster/libnl-3-dev_*.deb -sudo dpkg -i buildimage/target/debs/buster/libnl-genl-3-200_*.deb -sudo dpkg -i buildimage/target/debs/buster/libnl-genl-3-dev_*.deb -sudo dpkg -i buildimage/target/debs/buster/libnl-route-3-200_*.deb -sudo dpkg -i buildimage/target/debs/buster/libnl-route-3-dev_*.deb -sudo dpkg -i buildimage/target/debs/buster/libnl-nf-3-200_*.deb -sudo dpkg -i buildimage/target/debs/buster/libnl-nf-3-dev_*.deb -sudo dpkg -i buildimage/target/debs/buster/libnl-cli-3-200_*.deb -sudo dpkg -i buildimage/target/debs/buster/libnl-cli-3-dev_*.deb +mkdir -p scripts/common/sonic-swss-common-build/docker-sonic-vs/debs +cp *.deb scripts/common/sonic-swss-common-build/docker-sonic-vs/debs -pushd sonic-swss-common - -./autogen.sh -fakeroot debian/rules binary +docker load < buildimage/target/docker-sonic-vs.gz +pushd scripts/common/sonic-swss-common-build +docker build --no-cache -t docker-sonic-vs:${JOB_NAME##*/}.${BUILD_NUMBER} docker-sonic-vs popd -mkdir -p target -cp *.deb target/ - +docker save docker-sonic-vs:${JOB_NAME##*/}.${BUILD_NUMBER} | gzip -c > buildimage/target/docker-sonic-vs.gz diff --git a/scripts/common/sonic-swss-common-build/docker-sonic-vs/Dockerfile b/scripts/common/sonic-swss-common-build/docker-sonic-vs/Dockerfile new file mode 100644 index 0000000000..2ecd828a9f --- /dev/null +++ b/scripts/common/sonic-swss-common-build/docker-sonic-vs/Dockerfile @@ -0,0 +1,14 @@ +FROM docker-sonic-vs + +ADD ["debs", "/debs"] + +RUN dpkg -i /debs/libswsscommon_1.0.0_amd64.deb + +RUN dpkg -i /debs/libsaimetadata_1.0.0_amd64.deb +RUN dpkg -i /debs/libsairedis_1.0.0_amd64.deb +RUN dpkg -i /debs/libsaivs_1.0.0_amd64.deb +RUN dpkg -i /debs/syncd-vs_1.0.0_amd64.deb + +RUN dpkg -i /debs/swss_1.0.0_amd64.deb + +ENTRYPOINT ["/usr/bin/supervisord"] diff --git a/scripts/common/sonic-swss-common-build/docker_build_script.sh b/scripts/common/sonic-swss-common-build/docker_build_script.sh new file mode 100755 index 0000000000..eea56bce0d --- /dev/null +++ b/scripts/common/sonic-swss-common-build/docker_build_script.sh @@ -0,0 +1,64 @@ +#!/bin/bash -ex + +# Install swig +sudo apt-get install -y swig + +# Install HIREDIS +sudo apt-get install -y libhiredis0.14 libhiredis-dev + +# Install libnl3 +sudo dpkg -i buildimage/target/debs/buster/libnl-3-200_*.deb +sudo dpkg -i buildimage/target/debs/buster/libnl-3-dev_*.deb +sudo dpkg -i buildimage/target/debs/buster/libnl-genl-3-200_*.deb +sudo dpkg -i buildimage/target/debs/buster/libnl-genl-3-dev_*.deb +sudo dpkg -i buildimage/target/debs/buster/libnl-route-3-200_*.deb +sudo dpkg -i buildimage/target/debs/buster/libnl-route-3-dev_*.deb +sudo dpkg -i buildimage/target/debs/buster/libnl-nf-3-200_*.deb +sudo dpkg -i buildimage/target/debs/buster/libnl-nf-3-dev_*.deb +sudo dpkg -i buildimage/target/debs/buster/libnl-cli-3-200_*.deb +sudo dpkg -i buildimage/target/debs/buster/libnl-cli-3-dev_*.deb + +pushd sonic-swss-common + +./autogen.sh +fakeroot debian/rules binary + +# Install swsscommon packages for next steps of the build +sudo dpkg -i ../libswsscommon_*.deb +sudo dpkg -i ../libswsscommon-dev_*.deb + +popd + +# Build sairedis binaries +pushd sairedis + +./autogen.sh +fakeroot debian/rules binary-syncd-vs + +# Install sairedis packages for swss build +sudo dpkg -i ../libsaivs_*.deb +sudo dpkg -i ../libsaivs-dev_*.deb +sudo dpkg -i ../libsairedis_*.deb +sudo dpkg -i ../libsairedis-dev_*.deb +sudo dpkg -i ../libsaimetadata_*.deb +sudo dpkg -i ../libsaimetadata-dev_*.deb +sudo dpkg -i ../syncd-vs_*.deb + +popd + +# Install libteam for swss build +sudo apt-get install -y libdbus-1-3 +sudo dpkg -i buildimage/target/debs/buster/libteam5_*.deb +sudo dpkg -i buildimage/target/debs/buster/libteamdctl0_*.deb +sudo dpkg -i buildimage/target/debs/buster/libteam-utils_*.deb +sudo dpkg -i buildimage/target/debs/buster/libteam-dev_*.deb + +pushd swss + +./autogen.sh +fakeroot debian/rules binary + +popd + +mkdir -p target +cp *.deb target/ diff --git a/scripts/common/sonic-swss-common-build/docker_test_script.sh b/scripts/common/sonic-swss-common-build/docker_test_script.sh new file mode 100755 index 0000000000..fab64b0365 --- /dev/null +++ b/scripts/common/sonic-swss-common-build/docker_test_script.sh @@ -0,0 +1,29 @@ +#!/bin/bash -ex + +# Install Redis +sudo pip install Pympler==0.8 +sudo apt-get install -y redis-server +sudo sed -i 's/notify-keyspace-events ""/notify-keyspace-events AKE/' /etc/redis/redis.conf +sudo sed -ri 's/^# unixsocket/unixsocket/' /etc/redis/redis.conf +sudo sed -ri 's/^unixsocketperm .../unixsocketperm 777/' /etc/redis/redis.conf +sudo sed -ri 's/redis-server.sock/redis.sock/' /etc/redis/redis.conf +sudo service redis-server restart + +# Install libnl3 +sudo dpkg -i buildimage/target/debs/buster/libnl-3-200_*.deb +sudo dpkg -i buildimage/target/debs/buster/libnl-3-dev_*.deb +sudo dpkg -i buildimage/target/debs/buster/libnl-genl-3-200_*.deb +sudo dpkg -i buildimage/target/debs/buster/libnl-genl-3-dev_*.deb +sudo dpkg -i buildimage/target/debs/buster/libnl-route-3-200_*.deb +sudo dpkg -i buildimage/target/debs/buster/libnl-route-3-dev_*.deb +sudo dpkg -i buildimage/target/debs/buster/libnl-nf-3-200_*.deb +sudo dpkg -i buildimage/target/debs/buster/libnl-nf-3-dev_*.deb +sudo dpkg -i buildimage/target/debs/buster/libnl-cli-3-200_*.deb +sudo dpkg -i buildimage/target/debs/buster/libnl-cli-3-dev_*.deb + +sudo dpkg -i libswsscommon_*.deb +sudo dpkg -i python-swsscommon_*.deb + +cd sonic-swss-common + +sudo ./tests/tests && redis-cli FLUSHALL && pytest diff --git a/scripts/common/sonic-swss-common-build/test.sh b/scripts/common/sonic-swss-common-build/test.sh index b07ee8b1df..033f2fcdb3 100755 --- a/scripts/common/sonic-swss-common-build/test.sh +++ b/scripts/common/sonic-swss-common-build/test.sh @@ -1,17 +1,6 @@ #!/bin/bash -ex -# Install Redis -sudo pip install Pympler==0.8 -sudo apt-get install -y redis-server -sudo sed -i 's/notify-keyspace-events ""/notify-keyspace-events AKE/' /etc/redis/redis.conf -sudo sed -ri 's/^# unixsocket/unixsocket/' /etc/redis/redis.conf -sudo sed -ri 's/^unixsocketperm .../unixsocketperm 777/' /etc/redis/redis.conf -sudo sed -ri 's/redis-server.sock/redis.sock/' /etc/redis/redis.conf -sudo service redis-server restart +echo ${JOB_NAME##*/}.${BUILD_NUMBER} -sudo dpkg -i libswsscommon_*.deb -sudo dpkg -i python-swsscommon_*.deb - -cd sonic-swss-common - -sudo ./tests/tests && redis-cli FLUSHALL && pytest +docker pull sonicdev-microsoft.azurecr.io:443/sonic-slave-buster-johnar:latest +docker run --rm=true --privileged -v $(pwd):/sonic -w /sonic -i sonicdev-microsoft.azurecr.io:443/sonic-slave-buster-johnar:latest ./scripts/common/sonic-swss-common-build/docker_test_script.sh