Azure · Danny Allen (daall) · Aug 12, 2020 · Aug 12, 2020 · Aug 13, 2020 · Aug 13, 2020
diff --git a/jenkins/common/sonic-swss-common-build-pr/Jenkinsfile b/jenkins/common/sonic-swss-common-build-pr/Jenkinsfile
@@ -1,5 +1,9 @@
 pipeline {
-    agent { node { label 'sonic-slave-buster' } }
+    agent { node { label 'jenkins-vstest-workers' } }
+
+    options {
+        buildDiscarder(logRotator(artifactDaysToKeepStr: '', artifactNumToKeepStr: '', daysToKeepStr: '', numToKeepStr: '30'))
+    }
 
     stages {
         stage('Prepare') {
@@ -10,7 +14,27 @@ pipeline {
                               userRemoteConfigs: [[url: 'http://github.com/Azure/sonic-swss-common',
                                                    refspec: '+refs/pull/*:refs/remotes/origin/pr/*']]])
                 }
-                copyArtifacts(projectName: 'vs/buildimage-vs-all', filter: '**/*.deb', target: 'buildimage', flatten: false)
+
+                dir('sairedis') {
+                    checkout([$class: 'GitSCM',
+                              branches: [[name: '*/master']],
+                              extensions: [[$class: 'SubmoduleOption',
+                                            disableSubmodules: false,
+                                            parentCredentials: false,
+                                            recursiveSubmodules: true,
+                                            reference: '',
+                                            trackingSubmodules: false]],
+                              userRemoteConfigs: [[url: 'http://github.com/Azure/sonic-sairedis']]])
+
+                }
+
+                dir('swss') {
+                    checkout([$class: 'GitSCM',
+                              branches: [[name: '*/master']],
+                              userRemoteConfigs: [[url: 'https://github.com/Azure/sonic-swss']]])
+                }
+
+                copyArtifacts(projectName: 'vs/buildimage-vs-all', filter: '**/*.deb,**/docker-sonic-vs.gz', target: 'buildimage', flatten: false)
             }
         }
 
@@ -20,16 +44,28 @@ pipeline {
             }
         }
 
-        stage('Test') {
+        stage('swsscommon Unit Tests') {
             steps {
                 sh './scripts/common/sonic-swss-common-build/test.sh'
             }
         }
+
+        stage('Virtual Switch Tests') {
+            steps {
+                sh './scripts/vs/sonic-swss-build/test.sh'
+            }
+        }
     }
+
     post {
+        always {
+            archiveArtifacts(artifacts: 'target/*swsscommon*.deb, swss/tests/tr.xml')
+            junit(allowEmptyResults: true, keepLongStdio: true, testResults: 'swss/tests/tr.xml')
+            cleanWs(disableDeferredWipeout: false, deleteDirs: true, notFailBuild: true)
+        }
 
-        success {
-            archiveArtifacts(artifacts: 'target/*.deb')
+        failure {
+            archiveArtifacts(artifacts: 'buildimage/target/docker-sonic-vs.gz, **/*.log, target/var/log/*, swss/tests/log/**')
         }
     }
 }
diff --git a/jenkins/common/sonic-swss-common-build/Jenkinsfile b/jenkins/common/sonic-swss-common-build/Jenkinsfile
@@ -1,16 +1,12 @@
 pipeline {
-    agent { node { label 'sonic-slave-buster' } }
+    agent { node { label 'jenkins-vstest-workers' } }
 
     options {
         buildDiscarder(logRotator(artifactDaysToKeepStr: '', artifactNumToKeepStr: '', daysToKeepStr: '', numToKeepStr: '10'))
     }
 
-    environment {
-        SONIC_TEAM_WEBHOOK = credentials('public-jenkins-builder')
-    }
-
     triggers {
-        pollSCM('H/10 * * * *')
+        cron('H H/12 * * *')
     }
 
     stages {
@@ -21,7 +17,27 @@ pipeline {
                               branches: [[name: 'refs/heads/master']],
                               userRemoteConfigs: [[url: 'http://github.com/Azure/sonic-swss-common']]])
                 }
-                copyArtifacts(projectName: '../vs/buildimage-vs-all', filter: '**/*.deb', target: 'buildimage', flatten: false)
+
+                dir('sairedis') {
+                    checkout([$class: 'GitSCM',
+                              branches: [[name: '*/master']],
+                              extensions: [[$class: 'SubmoduleOption',
+                                            disableSubmodules: false,
+                                            parentCredentials: false,
+                                            recursiveSubmodules: true,
+                                            reference: '',
+                                            trackingSubmodules: false]],
+                              userRemoteConfigs: [[url: 'http://github.com/Azure/sonic-sairedis']]])
+
+                }
+
+                dir('swss') {
+                    checkout([$class: 'GitSCM',
+                              branches: [[name: '*/master']],
+                              userRemoteConfigs: [[url: 'https://github.com/Azure/sonic-swss']]])
+                }
+
+                copyArtifacts(projectName: 'vs/buildimage-vs-all', filter: '**/*.deb,**/docker-sonic-vs.gz', target: 'buildimage', flatten: false)
             }
         }
 
@@ -31,23 +47,30 @@ pipeline {
             }
         }
 
-        stage('Test') {
+        stage('swsscommon Unit Tests') {
             steps {
                 sh './scripts/common/sonic-swss-common-build/test.sh'
             }
         }
+
+        stage('Virtual Switch Tests') {
+            steps {
+                catchError(buildResult: 'SUCCESS', stageResult: 'UNSTABLE') {
+                    sh './scripts/vs/sonic-swss-build/test.sh'
+                }
+            }
+        }
     }
+
     post {
-        success {
-            archiveArtifacts(artifacts: 'target/*.deb')
+        always {
+            archiveArtifacts(artifacts: 'target/*swsscommon*.deb, swss/tests/tr.xml')
+            junit(allowEmptyResults: true, keepLongStdio: true, testResults: 'swss/tests/tr.xml')
+            cleanWs(disableDeferredWipeout: false, deleteDirs: true, notFailBuild: true)
         }
-        fixed {
-            slackSend(color:'#00FF00', message: "Build job back to normal: ${env.JOB_NAME} ${env.BUILD_NUMBER} (<${env.BUILD_URL}|Open>)")
-            office365ConnectorSend(webhookUrl: "${env.SONIC_TEAM_WEBHOOK}")
-        }
-        regression {
-            slackSend(color:'#FF0000', message: "Build job Regression: ${env.JOB_NAME} ${env.BUILD_NUMBER} (<${env.BUILD_URL}|Open>)")
-            office365ConnectorSend(webhookUrl: "${env.SONIC_TEAM_WEBHOOK}")
+
+        failure {
+            archiveArtifacts(artifacts: 'buildimage/target/docker-sonic-vs.gz, **/*.log, target/var/log/*, swss/tests/log/**')
         }
     }
 }
diff --git a/scripts/common/sonic-swss-common-build/build.sh b/scripts/common/sonic-swss-common-build/build.sh
@@ -1,30 +1,17 @@
 #!/bin/bash -ex
 
-# Install swig
-sudo apt-get install -y swig
+echo ${JOB_NAME##*/}.${BUILD_NUMBER}
 
-# Install HIREDIS
-sudo apt-get install -y libhiredis0.14 libhiredis-dev
+docker pull sonicdev-microsoft.azurecr.io:443/sonic-slave-buster-johnar:latest
+docker run --rm=true --privileged -v $(pwd):/sonic -w /sonic -i sonicdev-microsoft.azurecr.io:443/sonic-slave-buster-johnar:latest ./scripts/common/sonic-swss-common-build/docker_build_script.sh
 
-# Install libnl3
-sudo dpkg -i buildimage/target/debs/buster/libnl-3-200_*.deb
-sudo dpkg -i buildimage/target/debs/buster/libnl-3-dev_*.deb
-sudo dpkg -i buildimage/target/debs/buster/libnl-genl-3-200_*.deb
-sudo dpkg -i buildimage/target/debs/buster/libnl-genl-3-dev_*.deb
-sudo dpkg -i buildimage/target/debs/buster/libnl-route-3-200_*.deb
-sudo dpkg -i buildimage/target/debs/buster/libnl-route-3-dev_*.deb
-sudo dpkg -i buildimage/target/debs/buster/libnl-nf-3-200_*.deb
-sudo dpkg -i buildimage/target/debs/buster/libnl-nf-3-dev_*.deb
-sudo dpkg -i buildimage/target/debs/buster/libnl-cli-3-200_*.deb
-sudo dpkg -i buildimage/target/debs/buster/libnl-cli-3-dev_*.deb
+mkdir -p scripts/common/sonic-swss-common-build/docker-sonic-vs/debs
+cp *.deb scripts/common/sonic-swss-common-build/docker-sonic-vs/debs
 
-pushd sonic-swss-common
-
-./autogen.sh
-fakeroot debian/rules binary
+docker load < buildimage/target/docker-sonic-vs.gz
 
+pushd scripts/common/sonic-swss-common-build
+docker build --no-cache -t docker-sonic-vs:${JOB_NAME##*/}.${BUILD_NUMBER} docker-sonic-vs
 popd
 
-mkdir -p target
-cp *.deb target/
-
+docker save docker-sonic-vs:${JOB_NAME##*/}.${BUILD_NUMBER} | gzip -c > buildimage/target/docker-sonic-vs.gz
diff --git a/scripts/common/sonic-swss-common-build/docker-sonic-vs/Dockerfile b/scripts/common/sonic-swss-common-build/docker-sonic-vs/Dockerfile
@@ -0,0 +1,14 @@
+FROM docker-sonic-vs
+
+ADD ["debs", "/debs"]
+
+RUN dpkg -i /debs/libswsscommon_1.0.0_amd64.deb
+
+RUN dpkg -i /debs/libsaimetadata_1.0.0_amd64.deb
+RUN dpkg -i /debs/libsairedis_1.0.0_amd64.deb
+RUN dpkg -i /debs/libsaivs_1.0.0_amd64.deb
+RUN dpkg -i /debs/syncd-vs_1.0.0_amd64.deb
+
+RUN dpkg -i /debs/swss_1.0.0_amd64.deb
+
+ENTRYPOINT ["/usr/bin/supervisord"]
diff --git a/scripts/common/sonic-swss-common-build/docker_build_script.sh b/scripts/common/sonic-swss-common-build/docker_build_script.sh
@@ -0,0 +1,64 @@
+#!/bin/bash -ex
+
+# Install swig
+sudo apt-get install -y swig
+
+# Install HIREDIS
+sudo apt-get install -y libhiredis0.14 libhiredis-dev
+
+# Install libnl3
+sudo dpkg -i buildimage/target/debs/buster/libnl-3-200_*.deb
+sudo dpkg -i buildimage/target/debs/buster/libnl-3-dev_*.deb
+sudo dpkg -i buildimage/target/debs/buster/libnl-genl-3-200_*.deb
+sudo dpkg -i buildimage/target/debs/buster/libnl-genl-3-dev_*.deb
+sudo dpkg -i buildimage/target/debs/buster/libnl-route-3-200_*.deb
+sudo dpkg -i buildimage/target/debs/buster/libnl-route-3-dev_*.deb
+sudo dpkg -i buildimage/target/debs/buster/libnl-nf-3-200_*.deb
+sudo dpkg -i buildimage/target/debs/buster/libnl-nf-3-dev_*.deb
+sudo dpkg -i buildimage/target/debs/buster/libnl-cli-3-200_*.deb
+sudo dpkg -i buildimage/target/debs/buster/libnl-cli-3-dev_*.deb
+
+pushd sonic-swss-common
+
+./autogen.sh
+fakeroot debian/rules binary
+
+# Install swsscommon packages for next steps of the build
+sudo dpkg -i ../libswsscommon_*.deb
+sudo dpkg -i ../libswsscommon-dev_*.deb
+
+popd
+
+# Build sairedis binaries
+pushd sairedis
+
+./autogen.sh
+fakeroot debian/rules binary-syncd-vs
+
+# Install sairedis packages for swss build
+sudo dpkg -i ../libsaivs_*.deb
+sudo dpkg -i ../libsaivs-dev_*.deb
+sudo dpkg -i ../libsairedis_*.deb
+sudo dpkg -i ../libsairedis-dev_*.deb
+sudo dpkg -i ../libsaimetadata_*.deb
+sudo dpkg -i ../libsaimetadata-dev_*.deb
+sudo dpkg -i ../syncd-vs_*.deb
+
+popd
+
+# Install libteam for swss build
+sudo apt-get install -y libdbus-1-3
+sudo dpkg -i buildimage/target/debs/buster/libteam5_*.deb
+sudo dpkg -i buildimage/target/debs/buster/libteamdctl0_*.deb
+sudo dpkg -i buildimage/target/debs/buster/libteam-utils_*.deb
+sudo dpkg -i buildimage/target/debs/buster/libteam-dev_*.deb
+
+pushd swss
+
+./autogen.sh
+fakeroot debian/rules binary
+
+popd
+
+mkdir -p target
+cp *.deb target/
diff --git a/scripts/common/sonic-swss-common-build/docker_test_script.sh b/scripts/common/sonic-swss-common-build/docker_test_script.sh
@@ -0,0 +1,29 @@
+#!/bin/bash -ex
+
+# Install Redis
+sudo pip install Pympler==0.8
+sudo apt-get install -y redis-server
+sudo sed -i 's/notify-keyspace-events ""/notify-keyspace-events AKE/' /etc/redis/redis.conf
+sudo sed -ri 's/^# unixsocket/unixsocket/' /etc/redis/redis.conf
+sudo sed -ri 's/^unixsocketperm .../unixsocketperm 777/' /etc/redis/redis.conf
+sudo sed -ri 's/redis-server.sock/redis.sock/' /etc/redis/redis.conf
+sudo service redis-server restart
+
+# Install libnl3
+sudo dpkg -i buildimage/target/debs/buster/libnl-3-200_*.deb
+sudo dpkg -i buildimage/target/debs/buster/libnl-3-dev_*.deb
+sudo dpkg -i buildimage/target/debs/buster/libnl-genl-3-200_*.deb
+sudo dpkg -i buildimage/target/debs/buster/libnl-genl-3-dev_*.deb
+sudo dpkg -i buildimage/target/debs/buster/libnl-route-3-200_*.deb
+sudo dpkg -i buildimage/target/debs/buster/libnl-route-3-dev_*.deb
+sudo dpkg -i buildimage/target/debs/buster/libnl-nf-3-200_*.deb
+sudo dpkg -i buildimage/target/debs/buster/libnl-nf-3-dev_*.deb
+sudo dpkg -i buildimage/target/debs/buster/libnl-cli-3-200_*.deb
+sudo dpkg -i buildimage/target/debs/buster/libnl-cli-3-dev_*.deb
+
+sudo dpkg -i libswsscommon_*.deb
+sudo dpkg -i python-swsscommon_*.deb
+
+cd sonic-swss-common
+
+sudo ./tests/tests && redis-cli FLUSHALL && pytest
diff --git a/scripts/common/sonic-swss-common-build/test.sh b/scripts/common/sonic-swss-common-build/test.sh
@@ -1,17 +1,6 @@
 #!/bin/bash -ex
 
-# Install Redis
-sudo pip install Pympler==0.8
-sudo apt-get install -y redis-server
-sudo sed -i 's/notify-keyspace-events ""/notify-keyspace-events AKE/' /etc/redis/redis.conf
-sudo sed -ri 's/^# unixsocket/unixsocket/' /etc/redis/redis.conf
-sudo sed -ri 's/^unixsocketperm .../unixsocketperm 777/' /etc/redis/redis.conf
-sudo sed -ri 's/redis-server.sock/redis.sock/' /etc/redis/redis.conf
-sudo service redis-server restart
+echo ${JOB_NAME##*/}.${BUILD_NUMBER}
 
-sudo dpkg -i libswsscommon_*.deb
-sudo dpkg -i python-swsscommon_*.deb
-
-cd sonic-swss-common
-
-sudo ./tests/tests && redis-cli FLUSHALL && pytest
+docker pull sonicdev-microsoft.azurecr.io:443/sonic-slave-buster-johnar:latest
+docker run --rm=true --privileged -v $(pwd):/sonic -w /sonic -i sonicdev-microsoft.azurecr.io:443/sonic-slave-buster-johnar:latest ./scripts/common/sonic-swss-common-build/docker_test_script.sh