Jenkinsfile.ci

#!groovy


Ubuntu16 = "Ubuntu 16.04"
Ubuntu18 = "Ubuntu 18.04"
RedHat = "RedHat"

libindyBuildFinished = [(Ubuntu16): false, (Ubuntu18): false, (RedHat): false]

testing()

def testing() {
    stage('Static Analysis') {
        staticValidation()
    }
    stage('Testing') {
        parallel([
                'ubuntu-test' : { ubuntuTesting() },
                'android-test': { androidTesting() },
                'macos-test'  : { macosTesting() },
                'ios-test'    : { iosTesting() },
                'centos-test' : { centosTesting() },
                'windows-test': { windowsTesting() }
        ])
    }
}

def staticValidation() {
    parallel([
            'libindy'   : { runValdiationRust('libindy') },
            'libnullpay': { runValdiationRust('libnullpay') },
            'cli'       : { runValdiationRust('cli') }
    ])
}

def runValdiationRust(dir) {
    node('ubuntu') {
        try {
            cleanWs()
            checkout scm
            def testEnv = dockerBuild("static-validation-libindy", "libindy/ci/ubuntu.dockerfile libindy/ci")
            testEnv.inside {
                //FIXME waiting on IS-823
                //sh "cargo clippy --manifest-path ${dir}/Cargo.toml"
                sh "cargo clippy --manifest-path ${dir}/Cargo.toml -- -W clippy::style -W clippy::correctness -W clippy::complexity -W clippy::perf"
            }
        } finally {
            cleanWs()
        }
    }
}

def windowsTesting() {
    node('win2016') {
        stage('Windows Test') {
            def ws_path = "workspace/${env.JOB_NAME}".replace(' ', '_')
            ws(ws_path) {
                cleanWs()
                try {
                    echo "Windows Test: Checkout scm"
                    checkout scm

                    echo "Windows Test: Run Indy pool"
                    bat "docker -H $INDY_SDK_SERVER_IP build --build-arg pool_ip=$INDY_SDK_SERVER_IP -f ci/indy-pool.dockerfile -t indy_pool ci"
                    bat "docker -H $INDY_SDK_SERVER_IP run -d --network host --name indy_pool -p 9701-9708:9701-9708 indy_pool"

                    setupRust()

                    dir('libindy') {
                        echo "Windows Test: Download prebuilt dependencies"
                        bat 'wget -O prebuilt.zip "https://repo.sovrin.org/windows/libindy/deps/indy-sdk-deps.zip"'
                        bat 'unzip prebuilt.zip -d prebuilt'

                        echo "Windows Libindy Test: Build"
                        withEnv([
                                "INDY_PREBUILT_DEPS_DIR=$WORKSPACE\\libindy\\prebuilt",
                                "INDY_CRYPTO_PREBUILT_DEPS_DIR=$WORKSPACE\\libindy\\prebuilt",
                                "MILAGRO_DIR=$WORKSPACE\\libindy\\prebuilt",
                                "LIBZMQ_PREFIX=$WORKSPACE\\libindy\\prebuilt",
                                "SODIUM_LIB_DIR=$WORKSPACE\\libindy\\prebuilt",
                                "OPENSSL_DIR=$WORKSPACE\\libindy\\prebuilt",
                                "PATH=$WORKSPACE\\libindy\\prebuilt\\lib;$PATH",
                                "RUST_BACKTRACE=1"
                        ]) {
                            bat "cargo build --features fatal_warnings"
                            bat 'cargo test --no-run --features only_high_cases'

                            echo "Windows Libindy before Test: Deleting %TEMP\\indy_client"
                            bat "DEL /S /Q /F %TEMP%\\indy_client"
                            bat "DEL /S /Q /F %UserProfile%\\.indy_client"

                            echo "Windows Libindy Test: Run tests"
                            withEnv([
                                    "RUST_TEST_THREADS=1",
                                    "RUST_LOG=indy::=debug,zmq=trace",
                                    "TEST_POOL_IP=$INDY_SDK_SERVER_IP"
                            ]) {
                                bat 'cargo test --features only_high_cases'
                            }

                            echo "Windows Libindy after Test: Deleting %TEMP\\indy_client"
                            bat "DEL /S /Q /F %TEMP%\\indy_client"
                            bat "DEL /S /Q /F %UserProfile%\\.indy_client"

                        }
                    }

                    dir('libnullpay') {
                        echo "Windows Libnullpay Test: Build"
                        withEnv([
                                "RUST_BACKTRACE=1"
                        ]) {
                            bat "cargo build --features fatal_warnings"
                            bat "cargo test --no-run"

                            echo "Windows Libnullpay Test: Run tests"
                            withEnv([
                                    "RUST_TEST_THREADS=1",
                                    "RUST_LOG=indy::=debug,zmq=trace",
                                    "TEST_POOL_IP=$INDY_SDK_SERVER_IP",
                                    "PATH=$WORKSPACE\\libindy\\target\\debug;$PATH",
                            ]) {
                                bat "cargo test"
                            }
                        }
                    }

                    bat "PowerShell.exe \"Copy-Item $WORKSPACE/libnullpay/target/debug/nullpay.dll -Destination $WORKSPACE/cli\""

                    dir('cli') {
                        bat "PowerShell.exe \"&{(Get-Content -Path docker_pool_transactions_genesis -Raw) -replace '10\\.0\\.0\\.2','$INDY_SDK_SERVER_IP'} | Set-Content -Path docker_pool_transactions_genesis\""

                        def featuresArgs = ''
//                      def featuresArgs = '--features "nullpay_plugin"' // disabled because of IS-1109

                        echo "Windows Indy Cli Test: Build"
                        withEnv([
                                "RUST_BACKTRACE=1"
                        ]) {
                            bat "cargo test $featuresArgs --no-run"

                            echo "Windows Indy Cli Test: Run tests"
                            withEnv([
                                    "RUST_TEST_THREADS=1",
                                    "RUST_LOG=indy::=debug,zmq=trace",
                                    "TEST_POOL_IP=$INDY_SDK_SERVER_IP"
                            ]) {
                                bat "cargo test $featuresArgs"
                            }
                        }
                    }

                    //TODO wrappers testing

                } finally {
                    try {
                        echo "Windows Libindy after Test: Deleting %TEMP\\indy_client"
                        bat "DEL /S /Q /F %TEMP%\\indy_client"
                        bat "DEL /S /Q /F %UserProfile%\\.indy_client"
                    } catch (ignore) {
                    }
                    try {
                        bat "docker -H $INDY_SDK_SERVER_IP stop indy_pool"
                    } catch (ignore) {
                    }
                    try {
                        bat "docker -H $INDY_SDK_SERVER_IP rm indy_pool"
                    } catch (ignore) {
                    }
                    cleanWs()
                }
            }
            cleanWs()
        }
    }
}

def macosTesting() {
    node('macos') {
        cleanWs()
        stage('MacOS Test') {
            def pool_name = "indy_pool"
            try {
                echo "MacOS Test: Checkout scm"
                checkout scm

                setupRust()
                setupBrewPackages()

                macosRunPool(pool_name)
                macosCleanIndyDir()

                macosModuleBuildingAndTest('libindy', "--features only_high_cases")

                sh "cp libindy/target/debug/libindy.dylib libnullpay"
                sh "cp libindy/target/debug/libindy.dylib cli"

                macosModuleBuildingAndTest('libnullpay')

                sh "sed -i'.original' -e 's/10.0.0.2/127.0.0.1/g' cli/docker_pool_transactions_genesis"
                macosModuleBuildingAndTest('cli')

                //TODO wrappers testing
            } finally {
                macosStopPool(pool_name)
                macosCleanIndyDir()
                step([$class: 'WsCleanup'])
            }
        }
    }
}

def macosCleanIndyDir() {
    sh "rm -rf ~/.indy_client || true"
}

def macosModuleBuildingAndTest(library, testFeatures = "") {
    dir(library) {
        echo "MacOS ${library} Testing: Build"
        sh "LIBRARY_PATH=./ cargo build --features fatal_warnings"

        echo "MacOS ${library} Testing: Run Tests"
        sh "LIBRARY_PATH=./ cargo test --no-run ${testFeatures}"
        sh "LIBRARY_PATH=./ RUST_TEST_THREADS=1 cargo test ${testFeatures}"
    }
}

def androidTesting() {
    env_name = "android_on_linux"
    network_name = "pool_network"
    linuxModuleTesting("ci/android.dockerfile ci", env_name, network_name, this.&linuxAndroidOnEmulatorTesting, "android", "android", ["libindy": 'ci/ubuntu.dockerfile ci'])
}

def iosTesting() {

    def setupRustIOS = {
        sh "rustup target add x86_64-apple-ios"
    }

    def cleanSimulators = {
        sh '''
            xcrun simctl list devices | grep 'IndySDK' | awk '{print $4}' | sed -n  's/(\\(.*\\))/\\1/p'
            xcrun simctl list devices | grep 'IndySDK' | awk '{print $4}' | sed -n  's/(\\(.*\\))/\\1/p' | xargs xcrun simctl delete
        '''
    }

    def createSimulators = {
        sh '''
            xcrun simctl create "iPhone X IndySDK" "iPhone X" "11.2"
        '''
    }

    node('macos') {
        cleanWs()
        stage('iOS Test') {
            def pool_name = "indy_pool"
            try {
                echo "iOS Test: Checkout scm"
                checkout scm

                echo "iOS Test: setup rust, brew and simulators"

                //FIXME restore `setupRust()`
                //FIXME We downgrade rust for iOS build as 1.46.0 forces building cdylib crate that causes linking errors.
                //FIXME See https://github.com/rust-lang/rust/issues/79373 for details.
                //FIXME It is fixed in nightly already and we will switch to 1.49 after it is released.
                sh "rustup default 1.45.0"
                
                setupRustIOS()
                setupBrewPackages()
                cleanSimulators()
                createSimulators()

                withEnv([
                        "BUNDLE_PATH=$HOME/.gem.indy_sdk"
                ]) {
                    dir('wrappers/ios') {
                        echo "iOS Test: Installing ruby, bundler and gems"

                        sh '''
                            rbenv install -s $(cat .ruby-version)

                            gem install bundler

                            bundler install
                            bundler list
                        '''
                    }

                    dir('wrappers/ios/libindy-pod') {
                        echo "iOS Test: Installing dependencies (pods)"
                        sh "bundler exec pod install --repo-update"
                    }

                    echo "iOS Test: Running pool"
                    macosRunPool(pool_name)

                    sh 'chmod -R 777 ci'
                    echo "iOS Test: build libindy pod"
                    sh "ci/ios-build.sh libindy x86_64-apple-ios"

                    // FIXME replace this hack (manually rewrite installed pod)
                    sh "rm -f wrappers/ios/libindy-pod/Pods/libindy/*.[ah]"
                    sh "cp libindy/out_pod/*.[ah] wrappers/ios/libindy-pod/Pods/libindy"

                    // FIXME restore Xcode testing
//                    dir('wrappers/ios/libindy-pod') {
//                        echo "iOS Test: Testing"
//                        sh "xcodebuild test -workspace Indy.xcworkspace -scheme Indy-demo -destination 'platform=iOS Simulator,name=iPhone X IndySDK,OS=11.2'"
//                    }
                }
            } finally {
                try {
                    cleanSimulators()
                } catch (ignore) {
                }
                macosStopPool(pool_name)
                step([$class: 'WsCleanup'])
            }
        }
    }
}

def ubuntuTesting() {
    stage('Ubuntu Test') {
        parallel([
                "ubuntu:16.04": {
                    linuxTesting("ci/ubuntu.dockerfile ci", Ubuntu16, "pool_network", false)
                },
                "ubuntu:18.04": {
                    linuxTesting("ci/ubuntu18.dockerfile ci", Ubuntu18, "pool_network", true)
                }
        ])
    }
}

def centosTesting() {
    stage('RedHat Test') {
        linuxTesting("ci/centos.dockerfile ci", "RedHat", "pool_network", false)
    }
}

def linuxTesting(file, env_name, network_name, full_testing) {
    parallel([
            failFast                                  : true,
            "${env_name} Test: build and test libindy": {
                node('ubuntu') {
                    cleanWs()
                    linuxLibindyBuild(file, env_name, network_name, full_testing)
                }
            },
            "${env_name} Test: test other components" : {
                waitUntil {
                    libindyBuildFinished[env_name]
                }
                def jobs = [
                        "${env_name}-cli-test": {
                            linuxModuleTesting(file, env_name, network_name, this.&linuxCLITesting)
                        }
                ]
                if (full_testing) {
                    jobs["${env_name}wrappers-test"] = {
                        linuxModuleTesting(file, env_name, network_name, this.&linuxIndyWrappersTesting)
                    }
                }
                parallel(jobs)
            },
    ])
}

def linuxLibindyBuild(file, env_name, network_name, full_testing) {
    def poolInst
    try {
        echo "${env_name} Test: Checkout csm"
        checkout scm

        poolInst = openPool(env_name, network_name)

        def testEnv = buildTestEnv(file, env_name)

        testEnv.inside {
            echo "${env_name} Libindy Test: Build"
            sh '''
                cd libindy
                cargo build --features "fatal_warnings sodium_static"
            '''
        }

        sh "cp libindy/target/debug/libindy.so wrappers/java/lib"
        sh "cp libindy/target/debug/libindy.so wrappers/python"
        sh "cp libindy/target/debug/libindy.so wrappers/nodejs"
        sh "cp libindy/target/debug/libindy.so wrappers/rust"
        sh "cp libindy/target/debug/libindy.so cli"
        sh "cp libindy/target/debug/libindy.so libnullpay"

        testEnv.inside {
            echo "${env_name} Libnullpay Test: Build"
            sh '''
                cd libnullpay
                LIBRARY_PATH=./ cargo build --features fatal_warnings
            '''
        }

        sh "cp libnullpay/target/debug/libnullpay.so cli"

        stash includes: 'wrappers/java/lib/libindy.so', name: "LibindyJavaSO${env_name}"
        stash includes: 'wrappers/python/libindy.so', name: "LibindyPythonSO${env_name}"
        stash includes: 'wrappers/nodejs/libindy.so', name: "LibindyNodejsSO${env_name}"
        stash includes: 'wrappers/rust/libindy.so', name: "LibindyRustSO${env_name}"
        stash includes: 'cli/libindy.so', name: "LibindyCliSO${env_name}"
        stash includes: 'cli/libnullpay.so', name: "LibnullpayCliSO${env_name}"

        libindyBuildFinished[env_name] = true

        testEnv.inside("--network=${network_name}") {
            echo "${env_name} Libindy Test: Test Libindy submodules"

            for (component in ["libindy/indy-api-types", "libindy/indy-utils", "libindy/indy-wallet"]) {
                testLibindyComponent(component)
            }

            echo "${env_name} Libindy Test: Test"

            def features_args = full_testing ? "--features sodium_static" : "--features \"sodium_static only_high_cases\""

            sh """
                cd libindy
                RUST_BACKTRACE=1 cargo test --no-run ${features_args}
                RUST_BACKTRACE=1 RUST_LOG=indy::=debug,zmq=trace RUST_TEST_THREADS=1 TEST_POOL_IP=10.0.0.2 cargo test ${features_args}
            """
        }

        echo "${env_name} Libnullpay Test: Test"
        testEnv.inside("--network=${network_name}") {
            sh '''
                cd libnullpay
                LIBRARY_PATH=./ RUST_BACKTRACE=1 cargo test --no-run
                LD_LIBRARY_PATH=./:${LD_LIBRARY_PATH} RUST_BACKTRACE=1 RUST_LOG=indy::=debug,zmq=trace RUST_TEST_THREADS=1 TEST_POOL_IP=10.0.0.2 cargo test
            '''
        }
    }

    finally {
        closePool(env_name, network_name, poolInst)
    }
}

def testLibindyComponent(directory) {
    sh """
        cd ${directory}
        RUST_BACKTRACE=1 cargo test --no-run
        RUST_BACKTRACE=1 RUST_LOG=indy::=debug,zmq=trace RUST_TEST_THREADS=1 TEST_POOL_IP=10.0.0.2 cargo test
    """
}

def linuxModuleTesting(file, env_name, network_name, module_tester) {
    linuxModuleTesting(file, env_name, network_name, module_tester, "ubuntu")
}

def linuxModuleTesting(file, env_name, network_name, module_tester, node_label, image_name="libindy", prebuilt_images = [:]) {
    node(node_label) {
        cleanWs()
        def poolInst
        try {
            echo "${env_name} Test: Checkout csm"
            checkout scm

            poolInst = openPool(env_name, network_name)

            def testEnv = buildTestEnv(file, env_name, image_name, prebuilt_images)

            module_tester(env_name, network_name, testEnv)
        } finally {
            closePool(env_name, network_name, poolInst)
        }
    }
}

def linuxAndroidOnEmulatorTesting(env_name, network_name, testEnv) {
    sh 'chmod -R 777 libindy/ci/*.sh'
    sh 'chmod -R 777 libindy/*.sh'

    def emu_arch = "x86"

    testEnv.inside("--network=${network_name}") {
        for (directory in ["libindy", "libnullpay"]) {
            echo "${env_name} ${directory} Test: Test on ${emu_arch} emulator"
            sh "cd ${directory} && ./android.test.sh ${emu_arch}"
        }
    }
}

def linuxIndyWrappersTesting(env_name, network_name, testEnv) {
    linuxJavaTesting(env_name, network_name, testEnv)
    linuxPythonTesting(env_name, network_name, testEnv)
    linuxNodejsTesting(env_name, network_name, testEnv)
    linuxRustTesting(env_name, network_name, testEnv)
}

def linuxJavaTesting(env_name, network_name, testEnv) {
    unstash name: "LibindyJavaSO${env_name}"
    dir('wrappers/java') {
        testEnv.inside("--network=${network_name}") {
            echo "${env_name} Libindy Test: Test java wrapper"

            sh "TEST_POOL_IP=10.0.0.2 mvn clean test"
        }
    }
}

def linuxPythonTesting(env_name, network_name, testEnv) {
    unstash name: "LibindyPythonSO${env_name}"
    dir('wrappers/python') {
        testEnv.inside("--network=${network_name}") {
            echo "${env_name} Libindy Test: Test python wrapper"

            sh '''
                python3 -m pip install --user -e .[test]
                LD_LIBRARY_PATH=./:${LD_LIBRARY_PATH} TEST_POOL_IP=10.0.0.2 python3 -m pytest
            '''
        }
    }
}

def linuxNodejsTesting(env_name, network_name, testEnv) {
    unstash name: "LibindyNodejsSO${env_name}"
    testEnv.inside("--network=${network_name}") {
        echo "${env_name} Libindy Test: Test nodejs wrapper"

        sh '''
                cd wrappers/nodejs
                npm run prepare
                npm install
                LD_LIBRARY_PATH=./:${LD_LIBRARY_PATH} RUST_LOG=indy::=debug TEST_POOL_IP=10.0.0.2 npm test
            '''
    }
}

def linuxCLITesting(env_name, network_name, testEnv) {
    unstash name: "LibindyCliSO${env_name}"
    unstash name: "LibnullpayCliSO${env_name}"

    echo "${env_name} Indy Cli Test: tests"
    testEnv.inside("--network=${network_name}") {
        sh '''
            cd cli
            LIBRARY_PATH=./ RUST_BACKTRACE=1 cargo build --features fatal_warnings
            LIBRARY_PATH=./ RUST_BACKTRACE=1 cargo test --features "nullpay_plugin" --no-run
            LD_LIBRARY_PATH=./:${LD_LIBRARY_PATH} RUST_BACKTRACE=1 RUST_LOG=indy::=debug,zmq=trace RUST_TEST_THREADS=1 TEST_POOL_IP=10.0.0.2 cargo test --features "nullpay_plugin"
        '''
    }
}

def linuxRustTesting(env_name, network_name, testEnv) {
    unstash name: "LibindyRustSO${env_name}"

    dir('wrappers/rust') {
        testEnv.inside("--network=${network_name}") {
            echo "${env_name} Indy Rust wrapper Test: Build"
            sh 'LIBRARY_PATH=./ RUST_BACKTRACE=1 cargo test --no-run'

            echo "${env_name} Indy Rust wrupper Test: Run tests"
            sh 'LD_LIBRARY_PATH=./:${LD_LIBRARY_PATH} RUST_BACKTRACE=1 RUST_LOG=indy::=debug,zmq=trace RUST_TEST_THREADS=1 TEST_POOL_IP=10.0.0.2 cargo test'
        }
    }
}

def getBuildPoolVerOptions(pool_type, plenum_ver, anoncreds_ver, node_ver) {
    if (pool_type != null && plenum_ver != null && anoncreds_ver != null && node_ver != null) {
        return "--build-arg=indy_stream=${pool_type} --build-arg indy_plenum_ver=${plenum_ver} --build-arg indy_anoncreds_ver=${anoncreds_ver} --build-arg indy_node_ver=${node_ver}"
    } else {
        return ""
    }
}

def getUserUid() {
    return sh(returnStdout: true, script: 'id -u').trim()
}

def buildTestEnv(file, env_name, name="libindy", prebuilt_images=[:]) {
    dir('libindy') {
        echo "${env_name} Build  test environment"
        for (entry in mapToList(prebuilt_images)) {
            dockerBuild(entry[0], entry[1])
        }
        return dockerBuild(name, file)
    }
}

def dockerBuild(name, file = 'ci/ubuntu.dockerfile ci', customParams = '') {
    return docker.build("$name-test", "--build-arg uid=${getUserUid()} ${customParams} -f $file")
}

def macosRunPool(pool_name) {
    sh "docker rm -f ${pool_name} || true"
    sh "docker build -f ci/indy-pool.dockerfile -t ${pool_name} ci"
    sh "docker run -d --name ${pool_name} -p 9701-9708:9701-9708 ${pool_name}"
}

def macosStopPool(pool_name) {
    try {
        sh "docker stop ${pool_name}"
    } catch (ignore) {
    }
    try {
        sh "docker rm ${pool_name}"
    } catch (ignore) {
    }
}

def dockerClean(env_name, network_name) {
    try {
        try {
            sh returnStatus: true, script: "docker ps --format '{{.ID}}' --filter network=${network_name} | xargs docker rm -f"
        } catch (error) {
            echo "${env_name} Test: error while force clean-up network ${network_name} - ${error}"
        }
        try {
            echo "${env_name} Test: remove pool network ${network_name}"
            sh returnStatus: true, script: "docker network rm ${network_name}"
        } catch (error) {
            echo "${env_name} Test: error while delete ${network_name} - ${error}"
        }
    }
    finally {
        sh "docker container prune -f"
        sh "docker network prune -f"
    }
}

def openPool(env_name, network_name, pool_type = null, pool_ver = null, plenum_ver = null, anoncreds_ver = null, node_ver = null) {
    echo "${env_name} Test: Clean docker"
    dockerClean(env_name, network_name)

    echo "${env_name} Test: Create docker network (${network_name}) for nodes pool and test image"
    sh "docker network create --subnet=10.0.0.0/8 ${network_name}"

    echo "${env_name} Test: Build docker image for nodes pool ver. ${pool_ver}"
    echo "${env_name} Test: Building nodes pool for versions: plenum ${plenum_ver}, anoncreds ${anoncreds_ver}, node ${node_ver}"
    verOptions = getBuildPoolVerOptions(pool_type, plenum_ver, anoncreds_ver, node_ver)
    def poolEnv = dockerBuild("indy_pool_${pool_ver}", 'ci/indy-pool.dockerfile ci', "--build-arg pool_ip=10.0.0.2 ${verOptions}")
    echo "${env_name} Test: Run nodes pool"
    return poolEnv.run("--ip=\"10.0.0.2\" --network=${network_name}")
}

def closePool(env_name, network_name, poolInst) {
    echo "${env_name} Test: Cleanup"
    try {
        echo "${env_name} Test: stop pool"
        poolInst.stop()
    } catch (error) {
        echo "${env_name} Tests: error while stop pool ${error}"
    }
    finally {
        dockerClean(env_name, network_name)
    }
    step([$class: 'WsCleanup'])
}

def shell(command) {
    if (isUnix()) {
        sh command
    } else {
        bat command
    }
}

def setupRust() {
    shell("rustup default 1.46.0")
}

def setupBrewPackages() {
    sh "brew switch libsodium 1.0.12"
    sh "brew switch openssl 1.0.2q"
    sh "brew switch zeromq 4.2.3"
}

@NonCPS
List<List<?>> mapToList(Map map) {
    return map.collect { it ->
        [it.key, it.value]
    }
}