def setCommitStatus(String pStatus, String errorHint = '') {
    switch (pStatus) {
        case 'running':
            pState = 'running'
            pMessage = ""
            break
        case 'success':
            pState = 'success'
            pMessage = ":white_check_mark: ${env.JOB_NAME} :penguin: <b>Linux</b> SUCCEEDED :muscle:<br/>Build results: [Jenkins [${env.BUILD_DISPLAY_NAME}]](${env.RUN_DISPLAY_URL})<br/>Commit: ${env.GIT_COMMIT}"
            break
        case 'aborted':
            pState = 'canceled'
            pMessage = ":interrobang: ${env.JOB_NAME} :penguin: <b>Linux</b> ABORTED :confused:<br/>Build results: [Jenkins [${env.BUILD_DISPLAY_NAME}]](${env.RUN_DISPLAY_URL})<br/>Commit: ${env.GIT_COMMIT}"
            break
        case 'build_failure':
            pState = 'failed'
            pMessage = ":red_circle: ${env.JOB_NAME} :penguin: <b>Linux</b> FAILURE :worried:<br/>Build results: [Jenkins [${env.BUILD_DISPLAY_NAME}]](${env.RUN_DISPLAY_URL})<br/>Commit: ${env.GIT_COMMIT}"
            break
        case 'tests_failure':
            pState = 'failed'
            pMessage = "🟠 ${env.JOB_NAME} :penguin: <b>Linux</b> FAILURE :worried:<br/>Build results: [Jenkins [${env.BUILD_DISPLAY_NAME}]](${env.RUN_DISPLAY_URL})<br/>Commit: ${env.GIT_COMMIT}"
            break
        case 'unstable':
            pState = 'failed'
            pMessage = ":interrobang: ${env.JOB_NAME} :penguin: <b>Linux</b> UNSTABLE :confused:<br/>Build results: [Jenkins [${env.BUILD_DISPLAY_NAME}]](${env.RUN_DISPLAY_URL})<br/>Commit: ${env.GIT_COMMIT}"
            break
        default:
            pState = 'failed'
            pMessage = ":interrobang: ${env.JOB_NAME} :penguin: <b>Linux</b> UNKNOWN :confused:<br/>Build results: [Jenkins [${env.BUILD_DISPLAY_NAME}]](${env.RUN_DISPLAY_URL})<br/>Commit: ${env.GIT_COMMIT}"
    }
    if (pMessage) {
        if (errorHint) {
            pMessage += '<br/> ' + errorHint
        }
        addGitLabMRComment(comment: pMessage)
    }
    updateGitlabCommitStatus(name: "${PIPELINE_NAME}", state: pState)
}

def CUSTOM_TIMEOUT

pipeline {
    agent none
    options {
        buildDiscarder(logRotator(numToKeepStr: '135', daysToKeepStr: '21'))
        gitLabConnection('GitLabConnectionJenkins')
    }

    environment {
        PIPELINE_NAME = 'Build & test linux'
        BUILD_OPTIONS = ''
        TESTS_PARALLEL = ''
        APIURL_TO_TEST = ''
        GTEST_REPEAT = ''
        GTEST_FILTER = ''
        // SDK-5360 fu7-17 G6.S1: opt-in gates parsed from $gitlabTriggerPhrase (see
        // 'Get build and run paramters' stage). Default 'false' so the bench+tsan
        // stages are skipped on regular MRs (HR14: bench gate only on wsupload-touching).
        RUN_TSAN_SWEEP = 'false'
        RUN_BENCH_SWEEP = 'false'
    }

    stages {
        stage('Update Gitlab commit status') {
            agent any
            steps {
                updateGitlabCommitStatus(name: 'Build & test linux', state: 'running')
                script{
                    COMMIT_ID = env.GIT_COMMIT
                    println GIT_COMMIT
                }
            }
        }
        stage('Build'){
            agent { label 'linux && amd64' }
            stages {
                stage('clean previous runs'){
                    steps{
                        deleteDir()
                    }
                }
                stage('Get parameters'){
                    parallel{
                        stage('Get build and run paramters'){
                            steps {
                                script{
                                    BUILD_OPTIONS = sh(script: 'echo "$gitlabTriggerPhrase" | grep BUILD_OPTIONS | awk -F "BUILD_OPTIONS="  \'{print \$2}\' | cut -d"\\"" -f2 || :', returnStdout: true).trim()
                                    TESTS_PARALLEL = sh(script: 'echo "$gitlabTriggerPhrase" | grep "\\-\\-sequence" >/dev/null 2>&1 && echo "" || echo "--INSTANCES:10"', returnStdout: true).trim()
                                    GTEST_REPEAT = sh(script: 'echo "$gitlabTriggerPhrase" | grep --only-matching "\\-\\-gtest_repeat=[^ ]*" | awk -F "gtest_repeat="  \'{print "--gtest_repeat="\$2}\'|| :', returnStdout: true).trim()
                                    GTEST_FILTER = sh(script: 'echo "$gitlabTriggerPhrase" | grep --only-matching "\\-\\-gtest_filter=[^ ]*" | awk -F "gtest_filter="  \'{print "--gtest_filter="\$2}\'|| :', returnStdout: true).trim()
                                    CUSTOM_TIMEOUT = sh(script: 'echo "$gitlabTriggerPhrase" | grep --only-matching "\\-\\-timeout=[^ ]*" | awk -F "timeout="  \'{print \$2}\'|| :', returnStdout: true).trim()
                                    CUSTOM_TIMEOUT = CUSTOM_TIMEOUT? CUSTOM_TIMEOUT : "250"
                                    RUN_TSAN_SWEEP = sh(script: 'echo "$gitlabTriggerPhrase" | grep "\\-\\-tsan" >/dev/null 2>&1 && echo "true" || echo "false"', returnStdout: true).trim()
                                    RUN_BENCH_SWEEP = sh(script: 'echo "$gitlabTriggerPhrase" | grep "\\-\\-bench" >/dev/null 2>&1 && echo "true" || echo "false"', returnStdout: true).trim()
                                    println BUILD_OPTIONS
                                    println TESTS_PARALLEL
                                    println GTEST_REPEAT
                                    println GTEST_FILTER
                                    println CUSTOM_TIMEOUT
                                    println "RUN_TSAN_SWEEP=${RUN_TSAN_SWEEP}"
                                    println "RUN_BENCH_SWEEP=${RUN_BENCH_SWEEP}"
                                }
                            }
                        }

                        stage('Get API URL'){
                            steps {
                                script{
                                    APIURL_TO_TEST = sh(script: 'echo "$gitlabMergeRequestDescription" | grep USE_APIURL_TO_TEST | awk -F "USE_APIURL_TO_TEST="  \'{print \$2}\' | cut -d" " -f1', returnStdout: true).trim()
                                    println APIURL_TO_TEST
                                    if (APIURL_TO_TEST == ""){
                                        APIURL_TO_TEST = "https://g.api.mega.co.nz/"
                                        echo "APIURL_TO_TEST was not found on description so ${APIURL_TO_TEST} will be used by default"
                                    }
                                    echo "APIURL_TO_TEST will be ${APIURL_TO_TEST}"
                                }
                            }
                        }

                    }
                }

                stage('Checkout Linux'){
                    steps {
                        checkout([
                            $class: 'GitSCM',
                            branches: [[name: "origin/${env.gitlabSourceBranch}"]],
                            userRemoteConfigs: [[ url: "${env.GIT_URL_SDK}", credentialsId: "12492eb8-0278-4402-98f0-4412abfb65c1" ]],
                            extensions: [
                                [$class: "UserIdentity",name: "jenkins", email: "jenkins@jenkins"],
                                [$class: 'PreBuildMerge', options: [fastForwardMode: 'FF', mergeRemote: "origin", mergeStrategy: 'DEFAULT', mergeTarget: "${env.gitlabTargetBranch}"]]
                                ]
                        ])
                        script{
                            linux_sources_workspace = WORKSPACE
                            //linux_build_agent = "${NODE_NAME}"
                        }
                    }
                    post {
                        failure {
                            script {
                                setCommitStatus('build_failure', ':warning: Checkout failed')
                            }
                            echo 'Error: Checkout stage failed'
                        }
                    }
                }
                stage('Build Linux'){
                    options{
                        timeout(time: 120, unit: 'MINUTES')
                    }
                    environment{
                        VCPKGPATH = "/opt/vcpkg"
                        BUILD_DIR = "build_dir"
                        VCPKG_BINARY_SOURCES  = 'clear;x-aws,s3://vcpkg-cache/archives/,readwrite'
                        AWS_ACCESS_KEY_ID     = credentials('s4_access_key_id_vcpkg_cache')
                        AWS_SECRET_ACCESS_KEY = credentials('s4_secret_access_key_vcpkg_cache')
                        AWS_ENDPOINT_URL      = "https://s3.g.s4.mega.io"
                    }
                    steps{
                        dir("${linux_sources_workspace}") {
                            sh "rm -rf ${BUILD_DIR}; mkdir ${BUILD_DIR}"
                            //Build SDK
                            sh "echo Building SDK"
                            sh "cmake -DUSE_LIBUV=ON -DENABLE_QT_BINDINGS=ON -DENABLE_JAVA_BINDINGS=ON -DENABLE_CHAT=ON -DENABLE_MEDIA_FILE_METADATA=ON -DWITH_FUSE=1 -DCMAKE_BUILD_TYPE=Debug -DVCPKG_ROOT=${VCPKGPATH} \
                            ${BUILD_OPTIONS} -DCMAKE_VERBOSE_MAKEFILE=ON -S ${linux_sources_workspace} -B ${linux_sources_workspace}/${BUILD_DIR}"
                            sh "cmake --build ${linux_sources_workspace}/${BUILD_DIR} -j5"
                        }
                    }
                    post {
                        failure {
                            script {
                                setCommitStatus('build_failure', ':warning: Build failed. Tests won`t run')
                            }
                            echo "Error: Build stage failed. Tests won't run."
                        }
                    }
                }
                stage('Run Linux tests'){
                    environment {
                        MEGA_PWD = credentials('MEGA_PWD_DEFAULT')
                        MEGA_PWD_AUX = credentials('MEGA_PWD_DEFAULT')
                        MEGA_PWD_AUX2 = credentials('MEGA_PWD_DEFAULT')
                        MEGA_REAL_PWD=credentials('MEGA_REAL_PWD_TEST')
                        BUILD_DIR = "build_dir"
                    }
                    steps{
                        // S13 round-3 (Cluster J): the account-lock WAIT must not burn the test
                        // timeout — macOS/Windows nest lock{ timeout{ } } but linux nested them
                        // the other way round, so a job queued behind the single shared
                        // SDK_Concurrent_Test_Accounts resource could abort on CUSTOM_TIMEOUT
                        // without running a single test. Match the other platforms.
                        script {
                            def lockLabel = ''
                            if ("${APIURL_TO_TEST}" == 'https://g.api.mega.co.nz/') {
                                lockLabel = 'SDK_Concurrent_Test_Accounts'
                            } else  {
                                lockLabel = 'SDK_Concurrent_Test_Accounts_Staging'
                            }
                            lock(label: lockLabel, variable: 'ACCOUNTS_COMBINATION', quantity: 1, resourceSelectStrategy: "random", resource: null){
                                timeout(time: "${CUSTOM_TIMEOUT}", unit: 'MINUTES') {
                                    dir("${linux_sources_workspace}") {
                                        script{
                                            env.MEGA_EMAIL = "${env.ACCOUNTS_COMBINATION}"
                                            echo "${env.ACCOUNTS_COMBINATION}"
                                        }
                                        sh "echo Running tests"
                                        sh """#!/bin/bash
                                        set -x
                                        ulimit -c unlimited

                                        ${env.BUILD_DIR}/tests/unit/test_unit || FAILED=1

                                        if [ -z \"\$FAILED\" ]; then
                                            GFXWORKER_TEST_DIR=${env.BUILD_DIR}/tools/gfxworker/tests/integration/
                                            GFXWORKER_TEST_NAME=gfxworker_test_integration
                                            GFXWORKER_TEST_PATH=\"\$GFXWORKER_TEST_DIR\$GFXWORKER_TEST_NAME\"
                                            if [ -f \"\$GFXWORKER_TEST_PATH\" ]; then
                                                \$GFXWORKER_TEST_PATH || FAILED=2
                                                gzip -c gfxworker_test_integration.log > gfxworker_test_integration_${BUILD_ID}.log.gz || :
                                                rm gfxworker_test_integration.log || :
                                            fi
                                        fi

                                        # SDK-5360 fu8 S12 C(ii): provision the QaMixed real-media dataset on a
                                        # persistent per-agent path so SdkBenchmarkTest.QaMixedUpload RUNS instead
                                        # of skipping. HarvestQaMixedDataset is idempotent (skips files already
                                        # present at the right size), so this is a one-time download per agent.
                                        export MEGA_BENCH_UPLOAD_SOURCE_DIR=\"\${HOME}/mega_bench_dataset/qa_mixed\"
                                        if [ -z \"\$FAILED\" ]; then
                                            ${env.BUILD_DIR}/tests/integration/test_integration --FREEACCOUNTS --CI --USERAGENT:${env.USER_AGENT_TESTS_SDK} --APIURL:${APIURL_TO_TEST} --gtest_filter=SdkTest.HarvestQaMixedDataset || FAILED=2
                                        fi

                                        if [ -z \"\$FAILED\" ]; then
                                            if [ -z \"${TESTS_PARALLEL}\" ]; then
                                                # Sequential run
                                                ${env.BUILD_DIR}/tests/integration/test_integration --FREEACCOUNTS --CI --USERAGENT:${env.USER_AGENT_TESTS_SDK} --APIURL:${APIURL_TO_TEST} ${GTEST_FILTER} ${GTEST_REPEAT} &
                                                pid=\$!
                                                wait \$pid || FAILED=3
                                                if [ \"\$FAILED\" = \"3\" ]; then
                                                    cp ${linux_sources_workspace}/pid_\$pid/test_integration.log .
                                                fi
                                            else
                                                # Parallel run
                                                ${env.BUILD_DIR}/tests/integration/test_integration --FREEACCOUNTS --CI --USERAGENT:${env.USER_AGENT_TESTS_SDK} --APIURL:${APIURL_TO_TEST} ${GTEST_FILTER} ${GTEST_REPEAT} ${TESTS_PARALLEL} 2>&1 | tee tests.stdout
                                                [ \"\${PIPESTATUS[0]}\" != \"0\" ] && FAILED=3
                                            fi
                                        fi
                                        if [ -n \"\$FAILED\" ]; then
                                            if [ \"\$FAILED\" -le 2 ]; then
                                                coreFiles=core
                                            else # FAILED=3
                                                if [ -z \"${TESTS_PARALLEL}\" ]; then
                                                    # Sequential run
                                                    coreFiles=\"pid_\$pid/core\"
                                                else
                                                    # Parallel run
                                                    procFailed=`grep \"<< PROCESS\" tests.stdout | sed 's/.*PID:\\([0-9]*\\).*/\\1/'`
                                                    if [ -n \"\$procFailed\" ]; then
                                                        # Parallel run
                                                        for i in \$procFailed; do
                                                            coreFiles=\"\$coreFiles pid_\$i/core\"
                                                        done
                                                    fi
                                                fi
                                            fi
                                            if [ -n \"\$coreFiles\" ]; then
                                                maxTime=10
                                                startTime=`date +%s`
                                                coresProcessed=0
                                                coresTotal=`echo \$coreFiles | wc -w`
                                                # While there are pending cores
                                                while [ \$coresProcessed -lt \$coresTotal ] && [ \$( expr `date +%s` - \$startTime ) -lt \$maxTime ]; do
                                                    echo "Waiting for core dumps..."
                                                    sleep 1
                                                    for i in \$coreFiles; do
                                                        if [ -e \"\$i\" ] && [ -z \"\$( lsof \$i 2>/dev/null )\" ]; then
                                                            echo \"Processing core dump \$i :: \$(grep `echo \$i | sed 's#pid_\\([0-9].*\\)/core#\\1#'` tests.stdout)\"
                                                            echo thread apply all bt > backtrace
                                                            echo quit >> backtrace
                                                            [ \"\$FAILED\" = \"1\" ] && gdb -q ${env.BUILD_DIR}/tests/unit/test_unit \$i -x backtrace
                                                            [ \"\$FAILED\" = \"2\" ] && gdb -q \$GFXWORKER_TEST_PATH \$i -x backtrace
                                                            [ \"\$FAILED\" = \"3\" ] && gdb -q ${env.BUILD_DIR}/tests/integration/test_integration \$i -x backtrace
                                                            tar rf core.tar \$i
                                                            coresProcessed=`expr \$coresProcessed + 1`
                                                            coreFiles=`echo \$coreFiles | sed -e \"s#\$i##\"`
                                                        fi
                                                    done
                                                done
                                                if [ -e core.tar ]; then
                                                    [ \"\$FAILED\" = \"1\" ] && tar rf core.tar -C ${env.BUILD_DIR}/tests/unit/ test_unit
                                                    [ \"\$FAILED\" = \"2\" ] && tar rf core.tar -C \$GFXWORKER_TEST_DIR \$GFXWORKER_TEST_NAME
                                                    [ \"\$FAILED\" = \"3\" ] && tar rf core.tar -C ${env.BUILD_DIR}/tests/integration/ test_integration
                                                    gzip core.tar
                                                fi
                                            fi
                                        fi

                                        gzip -c test_integration.log > test_integration_${BUILD_ID}.log.gz || :
                                        rm test_integration.log || :
                                        # Archive bench_framework reports (fu7-5 G6 + fu7-18 G1). Two files per
                                        # test_integration PID, emitted when MEGA_BENCH_FRAMEWORK_ENABLED: a
                                        # per-iter JSONL (mid-run safe) plus a consolidated JSON written at
                                        # tear-down. Both live under pid_<PID>/bench_reports/.
                                        for benchReport in pid_*/bench_reports/bench_report_*.json pid_*/bench_reports/bench_report_*.jsonl; do
                                            [ -f \"\$benchReport\" ] && cp \"\$benchReport\" .
                                        done
                                        if [ -n \"\$FAILED\" ]; then
                                            exit \"\$FAILED\"
                                        fi
                                        """
                                    }
                                }
                            }
                        }
                    }
                    post {
                        always {
                            dir("${env.WORKSPACE}/src/fuse/supported/platform/posix/scripts") {
                                println "Attempting to clean up any stale MEGA-FS mounts"
                                sh "./cleanup.sh '${env.WORKSPACE}'"
                            }
                        }
                        failure {
                            script {
                                setCommitStatus('tests_failure', ':warning: Build succeed. Tests failed')
                            }
                            echo 'Error: Test stage failed'
                        }
                    }
                }
                stage('Build TSAN') {
                    when {
                        expression { return RUN_TSAN_SWEEP == 'true' }
                    }
                    options {
                        timeout(time: 120, unit: 'MINUTES')
                    }
                    environment {
                        VCPKGPATH = "/opt/vcpkg"
                        TSAN_BUILD_DIR = "build_dir_tsan"
                        VCPKG_BINARY_SOURCES  = 'clear;x-aws,s3://vcpkg-cache/archives/,readwrite'
                        AWS_ACCESS_KEY_ID     = credentials('s4_access_key_id_vcpkg_cache')
                        AWS_SECRET_ACCESS_KEY = credentials('s4_secret_access_key_vcpkg_cache')
                        AWS_ENDPOINT_URL      = "https://s3.g.s4.mega.io"
                    }
                    steps {
                        dir("${linux_sources_workspace}") {
                            sh "rm -rf ${TSAN_BUILD_DIR}; mkdir ${TSAN_BUILD_DIR}"
                            sh "echo Building SDK with TSAN"
                            sh "cmake -DUSE_LIBUV=ON -DENABLE_QT_BINDINGS=ON -DENABLE_JAVA_BINDINGS=ON -DENABLE_CHAT=ON -DENABLE_MEDIA_FILE_METADATA=ON -DWITH_FUSE=1 -DCMAKE_BUILD_TYPE=Debug -DENABLE_TSAN=ON -DVCPKG_ROOT=${VCPKGPATH} \
                            ${BUILD_OPTIONS} -DCMAKE_VERBOSE_MAKEFILE=ON -S ${linux_sources_workspace} -B ${linux_sources_workspace}/${TSAN_BUILD_DIR}"
                            sh "cmake --build ${linux_sources_workspace}/${TSAN_BUILD_DIR} -j5 --target test_integration test_unit"
                        }
                    }
                    post {
                        failure {
                            echo "Error: TSAN build stage failed. TSAN sweep skipped."
                        }
                    }
                }
                stage('Run Benchmarks') {
                    when {
                        expression { return RUN_BENCH_SWEEP == 'true' }
                    }
                    environment {
                        MEGA_PWD = credentials('MEGA_PWD_DEFAULT')
                        MEGA_PWD_AUX = credentials('MEGA_PWD_DEFAULT')
                        MEGA_PWD_AUX2 = credentials('MEGA_PWD_DEFAULT')
                        BUILD_DIR = "build_dir"
                        // SDK-5360 fu7-17 G6.S1: BENCH_FILTER is the fallback used when the user
                        // did NOT supply --gtest_filter= alongside --bench in the trigger phrase.
                        // When GTEST_FILTER is non-empty it wins; otherwise this default runs the
                        // full SdkBenchmarkTest.* cell.
                        BENCH_FILTER = "SdkBenchmarkTest.*"
                    }
                    steps {
                        timeout(time: 120, unit: 'MINUTES') {
                            script {
                                def lockLabel = ''
                                if ("${APIURL_TO_TEST}" == 'https://g.api.mega.co.nz/') {
                                    lockLabel = 'SDK_Concurrent_Test_Accounts'
                                } else  {
                                    lockLabel = 'SDK_Concurrent_Test_Accounts_Staging'
                                }
                                lock(label: lockLabel, variable: 'ACCOUNTS_COMBINATION', quantity: 1, resourceSelectStrategy: "random", resource: null){
                                    dir("${linux_sources_workspace}") {
                                        script{
                                            env.MEGA_EMAIL = "${env.ACCOUNTS_COMBINATION}"
                                        }
                                        sh """#!/bin/bash
                                        set -x
                                        ulimit -c unlimited
                                        # Bench cells emit bench_report_<PID>.json when MEGA_BENCH_FRAMEWORK_ENABLED=ON;
                                        # the framework cmake option must be enabled at configure time. The default MR
                                        # build does not enable it, so a separate configure+build under
                                        # ${env.BUILD_DIR}_bench keeps the bench artifacts isolated from the main build.
                                        rm -rf ${env.BUILD_DIR}_bench; mkdir ${env.BUILD_DIR}_bench
                                        cmake -DUSE_LIBUV=ON -DENABLE_QT_BINDINGS=ON -DENABLE_JAVA_BINDINGS=ON -DENABLE_CHAT=ON -DENABLE_MEDIA_FILE_METADATA=ON -DWITH_FUSE=1 -DCMAKE_BUILD_TYPE=Debug -DMEGA_BENCH_FRAMEWORK_ENABLED=ON -DVCPKG_ROOT=/opt/vcpkg \
                                            \${BUILD_OPTIONS} -DCMAKE_VERBOSE_MAKEFILE=ON -S ${linux_sources_workspace} -B ${linux_sources_workspace}/${env.BUILD_DIR}_bench
                                        cmake --build ${linux_sources_workspace}/${env.BUILD_DIR}_bench -j5 --target test_integration
                                        # SDK-5360 fu7-17 G6.S1: prefer user-supplied --gtest_filter= when present.
                                        # GTEST_FILTER (line ~85) already includes the --gtest_filter= prefix when set.
                                        # GTEST_REPEAT (line ~84) is appended unconditionally; empty when not given.
                                        if [ -n \"${GTEST_FILTER}\" ]; then
                                            BENCH_FILTER_ARG=\"${GTEST_FILTER}\"
                                        else
                                            BENCH_FILTER_ARG=\"--gtest_filter=${env.BENCH_FILTER}\"
                                        fi
                                        ${env.BUILD_DIR}_bench/tests/integration/test_integration --FREEACCOUNTS --CI --USERAGENT:${env.USER_AGENT_TESTS_SDK} --APIURL:${APIURL_TO_TEST} \"\$BENCH_FILTER_ARG\" ${GTEST_REPEAT} 2>&1 | tee bench_sweep.log || true
                                        # Copy bench reports (consolidated JSON + per-iter JSONL, fu7-18 G1) from
                                        # the bench-build's pid_<PID>/bench_reports/ dirs to top-level for archival.
                                        for benchReport in ${env.BUILD_DIR}_bench/pid_*/bench_reports/bench_report_*.json ${env.BUILD_DIR}_bench/pid_*/bench_reports/bench_report_*.jsonl pid_*/bench_reports/bench_report_*.json pid_*/bench_reports/bench_report_*.jsonl; do
                                            [ -f "\$benchReport" ] && cp "\$benchReport" .
                                        done
                                        # SDK-5360 fu7-12 G5.c: per-cell bench log split. csplit on gtest [ RUN ]
                                        # markers, rename per-cell, tar.gz into bench_logs_${BUILD_ID}.tar.gz so
                                        # reviewers can wget one cell directly (analogous to gfx-log separation).
                                        mkdir -p bench_logs_${BUILD_ID} || :
                                        if [ -s bench_sweep.log ]; then
                                            csplit -z -s -b '%04d' -f bench_logs_${BUILD_ID}/_split_ bench_sweep.log '/\\[ RUN      \\]/' '{*}' || :
                                            for sf in bench_logs_${BUILD_ID}/_split_*; do
                                                [ -s "\$sf" ] || { rm -f "\$sf"; continue; }
                                                cellName=\$(grep -m1 -oE '\\[ RUN      \\] [A-Za-z0-9_.]+' "\$sf" | awk '{print \$4}' | tr '.' '_' || echo 'preamble')
                                                [ -n "\$cellName" ] || cellName="preamble"
                                                mv "\$sf" "bench_logs_${BUILD_ID}/\${cellName}.log" || :
                                            done
                                        fi
                                        tar -czf bench_logs_${BUILD_ID}.tar.gz bench_logs_${BUILD_ID} || :
                                        rm -rf bench_logs_${BUILD_ID} || :
                                        gzip -c bench_sweep.log > bench_sweep_${BUILD_ID}.log.gz || :
                                        rm bench_sweep.log || :
                                        """
                                    }
                                }
                            }
                        }
                    }
                    post {
                        failure {
                            echo 'Error: Bench sweep failed'
                        }
                    }
                }
                stage('Run TSAN sweep') {
                    when {
                        expression { return RUN_TSAN_SWEEP == 'true' }
                    }
                    environment {
                        MEGA_PWD = credentials('MEGA_PWD_DEFAULT')
                        MEGA_PWD_AUX = credentials('MEGA_PWD_DEFAULT')
                        MEGA_PWD_AUX2 = credentials('MEGA_PWD_DEFAULT')
                        TSAN_BUILD_DIR = "build_dir_tsan"
                        TSAN_OPTIONS = "halt_on_error=0:second_deadlock_stack=1:history_size=7:report_thread_leaks=0"
                        // SDK-5360 fu7-12 G5.b: current WS-upload surface cells (post-fu7-7 rename;
                        // pre-fu7-7 SdkTest.* names removed). HR23 strict: any NEW race on
                        // surface fields (mChunksInFlight, mUploadingFile, etc.) fails the stage.
                        // SDK-5360 fu7-17 G6.S1: this is the fallback used when the user did
                        // NOT supply --gtest_filter= alongside --tsan. User filter wins below.
                        // SDK-5360 fu8 S12: + the two watchdog cells (CI TSAN previously never covered
                        // the watchdog/pause recovery paths; S11 carry-over ruled in by the user).
                        TSAN_SURFACE_FILTER = "SdkWsUploadTest.ActivePoolUsesParallelConnections:SdkWsUploadTest.InvalidPinnedSessionFallsBackToFreshSession:SdkWsUploadTest.OverquotaDuringTransfer:SdkWsUploadTest.RepeatedPauseResumeMixedPools:SdkWsUploadTest.AckStallForceReconnectsHungConnection:SdkWsUploadTest.TailCompletionWatchdogRecoversDroppedCompletion:SdkWsUploadTest.QuotaCrossPoolHoldDoesNotBlockOtherPool:SdkWsUploadTest.QuotaAccountRedOverquotaCoexistsWithPredictiveHolds:SdkWsUploadTest.LocallogoutCancelsBlackholedHandshake"
                        // SDK-6476: hermetic unit cells written for this sweep (the CurlHttpIO
                        // proxy/DNS/user-agent config race is invisible to the regular test_unit run).
                        // Any race report from them fails the stage.
                        TSAN_UNIT_FILTER = "CurlHttpIoConfig.*"
                    }
                    steps {
                        timeout(time: 90, unit: 'MINUTES') {
                            script {
                                def lockLabel = ''
                                if ("${APIURL_TO_TEST}" == 'https://g.api.mega.co.nz/') {
                                    lockLabel = 'SDK_Concurrent_Test_Accounts'
                                } else  {
                                    lockLabel = 'SDK_Concurrent_Test_Accounts_Staging'
                                }
                                lock(label: lockLabel, variable: 'ACCOUNTS_COMBINATION', quantity: 1, resourceSelectStrategy: "random", resource: null){
                                    dir("${linux_sources_workspace}") {
                                        script{
                                            env.MEGA_EMAIL = "${env.ACCOUNTS_COMBINATION}"
                                        }
                                        sh """#!/bin/bash
                                        set -x
                                        ulimit -c unlimited
                                        # SDK-6476: TSAN-oracle unit cells first (hermetic, no accounts): any race fails the
                                        # stage before the account-bound integration sweep starts. One log covers both binaries.
                                        ${env.TSAN_BUILD_DIR}/tests/unit/test_unit \"--gtest_filter=${TSAN_UNIT_FILTER}\" 2>&1 | tee tsan_sweep.log || true
                                        UNIT_RACES=\$(grep -c "WARNING: ThreadSanitizer" tsan_sweep.log || :)
                                        echo "===TSAN UNIT CELLS (${TSAN_UNIT_FILTER})=== races: \${UNIT_RACES:-0}"
                                        if [ \"\${UNIT_RACES:-0}\" != \"0\" ]; then
                                            echo "[ERROR] TSAN race in the unit cells" >&2
                                            exit 1
                                        fi
                                        # SDK-5360 fu7-17 G6.S1: prefer user-supplied --gtest_filter= when present.
                                        # GTEST_FILTER already includes the --gtest_filter= prefix when set.
                                        if [ -n \"${GTEST_FILTER}\" ]; then
                                            TSAN_FILTER_ARG=\"${GTEST_FILTER}\"
                                        else
                                            TSAN_FILTER_ARG=\"--gtest_filter=${TSAN_SURFACE_FILTER}\"
                                        fi
                                        ${env.TSAN_BUILD_DIR}/tests/integration/test_integration --FREEACCOUNTS --CI --USERAGENT:${env.USER_AGENT_TESTS_SDK} --APIURL:${APIURL_TO_TEST} \"\$TSAN_FILTER_ARG\" ${GTEST_REPEAT} 2>&1 | tee -a tsan_sweep.log || true
                                        # Surface race summary: total reports + WS-upload surface field hits
                                        echo "===TSAN SUMMARY==="
                                        grep -cE "WARNING: ThreadSanitizer: data race" tsan_sweep.log || :
                                        echo "===WS-UPLOAD SURFACE FIELDS==="
                                        for fld in mChunksInFlight mNumChunksInFlight mUploadingFile WsConn::mLastActive WsConn::mPausedByServerUntilDs WsConn::mReadyState WsUploadFile mWorkGeneration; do
                                            cnt=\$(grep -cE "\$fld" tsan_sweep.log || :)
                                            echo "\$fld: \$cnt"
                                            if [ \"\$cnt\" != \"0\" ]; then
                                                echo "[ERROR] new WS-upload surface race on \$fld" >&2
                                                exit 1
                                            fi
                                        done
                                        # SDK-5360 fu7-17 G6.S1: per-cell TSAN log split (same shape as bench).
                                        # csplit on gtest [ RUN ] markers, rename per-cell, tar.gz so reviewers
                                        # can wget one cell's TSAN output directly.
                                        mkdir -p tsan_logs_${BUILD_ID} || :
                                        if [ -s tsan_sweep.log ]; then
                                            csplit -z -s -b '%04d' -f tsan_logs_${BUILD_ID}/_split_ tsan_sweep.log '/\\[ RUN      \\]/' '{*}' || :
                                            for sf in tsan_logs_${BUILD_ID}/_split_*; do
                                                [ -s "\$sf" ] || { rm -f "\$sf"; continue; }
                                                cellName=\$(grep -m1 -oE '\\[ RUN      \\] [A-Za-z0-9_.]+' "\$sf" | awk '{print \$4}' | tr '.' '_' || echo 'preamble')
                                                [ -n "\$cellName" ] || cellName="preamble"
                                                mv "\$sf" "tsan_logs_${BUILD_ID}/\${cellName}.log" || :
                                            done
                                        fi
                                        tar -czf tsan_logs_${BUILD_ID}.tar.gz tsan_logs_${BUILD_ID} || :
                                        rm -rf tsan_logs_${BUILD_ID} || :
                                        gzip -c tsan_sweep.log > tsan_sweep_${BUILD_ID}.log.gz || :
                                        rm tsan_sweep.log || :
                                        """
                                    }
                                }
                            }
                        }
                    }
                    post {
                        failure {
                            echo 'Error: TSAN sweep failed — NEW WS-upload surface race detected'
                        }
                    }
                }
            }
            post{
                always {
                    dir("${env.WORKSPACE}/src/fuse/supported/platform/posix/scripts") {
                        echo "Final cleanup of any stale MEGA-FS mounts"
                        sh "./cleanup.sh '${env.WORKSPACE}'"
                    }
                    archiveArtifacts artifacts: '*test_integration*, core.tar.gz, bench_report_*.json, bench_report_*.jsonl, tsan_sweep*.log.gz, bench_sweep*.log.gz, bench_logs_*.tar.gz, tsan_logs_*.tar.gz', fingerprint: true, allowEmptyArchive: true
                    deleteDir()
                }
            }
        }
    }
    post {
        success {
            updateGitlabCommitStatus(name: 'Build & test linux', state: 'success')
            addGitLabMRComment(comment: ":white_check_mark: ${currentBuild.projectName} :penguin: <b>Linux</b> SUCCEEDED :muscle:<br/>Build results: [Jenkins [${currentBuild.displayName}]](${currentBuild.absoluteUrl})<br/>Commit: ${COMMIT_ID}" )
        }
        unstable {
            updateGitlabCommitStatus(name: 'Build & test linux', state: 'failed')
            addGitLabMRComment(comment: ":interrobang: ${currentBuild.projectName} :penguin: <b>Linux</b> UNSTABLE  :confused:<br/>Build results: [Jenkins [${currentBuild.displayName}]](${currentBuild.absoluteUrl})<br/>Commit: ${COMMIT_ID}" )      
        }
        aborted {
            updateGitlabCommitStatus(name: 'Build & test linux', state: 'canceled')
            addGitLabMRComment(comment: ":interrobang: ${currentBuild.projectName} :penguin: <b>Linux</b> ABORTED  :confused:<br/>Build results: [Jenkins [${currentBuild.displayName}]](${currentBuild.absoluteUrl})<br/>Commit: ${COMMIT_ID}" )
        }
    }
}
