Started by user Jenkins Admin Obtained pipelines/pingcap/tiflow/latest/pull_cdc_integration_pulsar_test.groovy from git https://github.com/PingCAP-QE/ci.git Loading library tipipeline@main Library tipipeline@main is cached. Copying from home. [Pipeline] Start of Pipeline [Pipeline] readJSON [Pipeline] readTrusted Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_pulsar_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-2vb7-5sbdh Agent pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-2vb7-5sbdh is provisioned from template pingcap_tiflow_pull_cdc_integration_pulsar_test_3484-2vb7n-6b439 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/3484/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/3484/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "a60f44058dcf6e401fe63304d06ba740044ca15d" jenkins/label: "pingcap_tiflow_pull_cdc_integration_pulsar_test_3484-2vb7n" name: "pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-2vb7-5sbdh" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/rocky8_golang-1.23:latest" name: "golang" resources: limits: memory: "32Gi" cpu: "12" requests: memory: "32Gi" cpu: "12" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-2vb7-5sbdh" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-2vb7-5sbdh" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-2vb7-5sbdh in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test [Pipeline] { [Pipeline] stage [Pipeline] { (Declarative: Checkout SCM) [Pipeline] checkout The recommended git tool is: git No credentials specified Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Avoid second fetch Checking out Revision b2a59508c11c86ba701c4af25fc536a9475b82c6 (origin/main) > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f b2a59508c11c86ba701c4af25fc536a9475b82c6 # timeout=10 Commit message: "feat(prow-jobs/pingcap-qe/ci): add periodic job to crawl pull requests and issues (#3449)" First time build. Skipping changelog. [Pipeline] } [Pipeline] // stage [Pipeline] withEnv [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] timeout Timeout set to expire in 1 hr 0 min [Pipeline] { [Pipeline] stage [Pipeline] { (Debug info) [Pipeline] sh > git rev-list --no-walk 3e31ec89ba643faa364a78dbc7f062bb55f9c486 # timeout=10 + printenv GIT_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/3484/display/redirect JENKINS_URL=https://do.pingcap.net/jenkins/ EXECUTOR_NUMBER=0 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_pulsar_test","buildid":"1908056220913111045","prowjobid":"b4bfd23f-f137-430c-9c71-ddcc89dd7a6d","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","base_link":"https://github.com/pingcap/tiflow/commit/4c631d5951854ac91765b729fe3849d8e8d5ed8a","pulls":[{"number":12118,"author":"dveeden","sha":"a306f48949e037965a7c1963f500f7e0abbfba51","title":"dm: upgrade go-mysql to v1.12.0","head_ref":"new_gomysql","link":"https://github.com/pingcap/tiflow/pull/12118","commit_link":"https://github.com/pingcap/tiflow/pull/12118/commits/a306f48949e037965a7c1963f500f7e0abbfba51","author_link":"https://github.com/dveeden"}]}} _=/usr/bin/printenv TZ=Asia/Shanghai BUILD_ID=1908056220913111045 POD_LABEL=pingcap_tiflow_pull_cdc_integration_pulsar_test_3484-2vb7n HOSTNAME=pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-2vb7-5sbdh OLDPWD=/home/jenkins RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/3484/display/redirect?page=changes JENKINS_NODE_COOKIE=e4870882-2011-4d1a-8b6b-b9ca4d507925 GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct JAVA_HOME=/usr/lib/jvm/jre-openjdk JOB_BASE_NAME=pull_cdc_integration_pulsar_test WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test@tmp KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true GIT_URL=https://github.com/PingCAP-QE/ci.git CLASSPATH= KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 FILE_SERVER_URL=http://fileserver.pingcap.net POD_CONTAINER=golang GOPATH=/go BUILD_NUMBER=3484 USE_BAZEL_VERSION=6.5.0 KUBERNETES_PORT=tcp://10.233.0.1:443 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test HUDSON_URL=https://do.pingcap.net/jenkins/ HOME=/home/jenkins NODE_NAME=pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-2vb7-5sbdh HUDSON_SERVER_COOKIE=83ef27fe9acccc92 JENKINS_HOME=/var/jenkins_home JOB_NAME=pingcap/tiflow/pull_cdc_integration_pulsar_test RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/3484/display/redirect?page=tests KUBERNETES_SERVICE_PORT_HTTPS=443 KUBERNETES_PORT_443_TCP_PORT=443 GIT_PREVIOUS_COMMIT=3e31ec89ba643faa364a78dbc7f062bb55f9c486 HUDSON_HOME=/var/jenkins_home JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/display/redirect PROW_JOB_ID=b4bfd23f-f137-430c-9c71-ddcc89dd7a6d KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/3484/display/redirect?page=artifacts BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/3484/ TERM=xterm STAGE_NAME=Debug info JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/ BUILD_DISPLAY_NAME=#3484 SHLVL=3 GIT_BRANCH=origin/main BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-3484 KUBERNETES_SERVICE_PORT=443 NODE_LABELS=pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-2vb7-5sbdh pingcap_tiflow_pull_cdc_integration_pulsar_test_3484-2vb7n PATH=/go/bin:/root/.cargo/bin:/usr/local/go/bin/:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/gradle-7.4.2/bin:/opt/apache-maven-3.8.8/bin:/usr/local/pulsar/bin GIT_PREVIOUS_SUCCESSFUL_COMMIT=3e31ec89ba643faa364a78dbc7f062bb55f9c486 KUBERNETES_SERVICE_HOST=10.233.0.1 JENKINS_SERVER_COOKIE=durable-100f7bb7add7cfb003dff5768bf20348b91a6b6cef38154d1625c661ef800a11 + echo ------------------------- ------------------------- + go env GO111MODULE='' GOARCH='amd64' GOBIN='' GOCACHE='/home/jenkins/.cache/go-build' GOENV='/home/jenkins/.config/go/env' GOEXE='' GOEXPERIMENT='' GOFLAGS='' GOHOSTARCH='amd64' GOHOSTOS='linux' GOINSECURE='' GOMODCACHE='/go/pkg/mod' GONOPROXY='' GONOSUMDB='' GOOS='linux' GOPATH='/go' GOPRIVATE='' GOPROXY='http://goproxy.apps.svc,https://proxy.golang.org,direct' GOROOT='/usr/local/go' GOSUMDB='sum.golang.org' GOTMPDIR='' GOTOOLCHAIN='auto' GOTOOLDIR='/usr/local/go/pkg/tool/linux_amd64' GOVCS='' GOVERSION='go1.23.3' GODEBUG='' GOTELEMETRY='local' GOTELEMETRYDIR='/home/jenkins/.config/go/telemetry' GCCGO='gccgo' GOAMD64='v1' AR='ar' CC='gcc' CXX='g++' CGO_ENABLED='1' GOMOD='/dev/null' GOWORK='' CGO_CFLAGS='-O2 -g' CGO_CPPFLAGS='' CGO_CXXFLAGS='-O2 -g' CGO_FFLAGS='-O2 -g' CGO_LDFLAGS='-O2 -g' PKG_CONFIG='pkg-config' GOGCCFLAGS='-fPIC -m64 -pthread -Wl,--no-gc-sections -fmessage-length=0 -ffile-prefix-map=/tmp/go-build2812431546=/tmp/go-build -gno-record-gcc-switches' + echo ------------------------- ------------------------- + echo 'debug command: kubectl -n jenkins-tiflow exec -ti pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-2vb7-5sbdh bash' debug command: kubectl -n jenkins-tiflow exec -ti pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-2vb7-5sbdh bash [Pipeline] container [Pipeline] { [Pipeline] sh + dig github.com ; <<>> DiG 9.18.16 <<>> github.com ;; global options: +cmd ;; Got answer: ;; ->>HEADER<<- opcode: QUERY, status: NOERROR, id: 10447 ;; flags: qr aa rd ra; QUERY: 1, ANSWER: 2, AUTHORITY: 0, ADDITIONAL: 1 ;; OPT PSEUDOSECTION: ; EDNS: version: 0, flags:; udp: 1232 ; COOKIE: 334d5930e1fbdd72 (echoed) ;; QUESTION SECTION: ;github.com. IN A ;; ANSWER SECTION: github.com. 27 IN A 20.27.177.113 github.com. 27 IN A 20.205.243.166 ;; Query time: 0 msec ;; SERVER: 169.254.25.10#53(169.254.25.10) (UDP) ;; WHEN: Fri Apr 04 07:17:36 UTC 2025 ;; MSG SIZE rcvd: 103 [Pipeline] script [Pipeline] { [Pipeline] } [Pipeline] // script [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // stage [Pipeline] stage [Pipeline] { (Check diff files) [Pipeline] container [Pipeline] { [Pipeline] script [Pipeline] { [Pipeline] withCredentials Masking supported pattern matches of $token [Pipeline] { [Pipeline] httpRequest Warning: A secret was passed to "httpRequest" using Groovy String interpolation, which is insecure. Affected argument(s) used the following variable(s): [token] See https://jenkins.io/redirect/groovy-string-interpolation for details. HttpMethod: GET URL: https://api.github.com/repos/pingcap/tiflow/pulls/12118/files?page=1&per_page=100 Content-Type: application/json Authorization: ***** Sending request to url: https://api.github.com/repos/pingcap/tiflow/pulls/12118/files?page=1&per_page=100 Response Code: HTTP/1.1 200 OK Success: Status code 200 is in the accepted range: 100:399 [Pipeline] httpRequest Warning: A secret was passed to "httpRequest" using Groovy String interpolation, which is insecure. Affected argument(s) used the following variable(s): [token] See https://jenkins.io/redirect/groovy-string-interpolation for details. HttpMethod: GET URL: https://api.github.com/repos/pingcap/tiflow/pulls/12118/files?page=2&per_page=100 Content-Type: application/json Authorization: ***** Sending request to url: https://api.github.com/repos/pingcap/tiflow/pulls/12118/files?page=2&per_page=100 Response Code: HTTP/1.1 200 OK Success: Status code 200 is in the accepted range: 100:399 [Pipeline] } [Pipeline] // withCredentials [Pipeline] echo pr_diff_files: [go.mod, go.sum] [Pipeline] echo diff file not matched: go.mod [Pipeline] } [Pipeline] // script [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // stage [Pipeline] stage [Pipeline] { (Checkout) [Pipeline] timeout Timeout set to expire in 10 min [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow [Pipeline] { [Pipeline] cache Cache restored successfully (git/pingcap/tiflow/rev-4c631d5-a306f48) 192337920 bytes in 0.89 secs (217000317 bytes/sec) [Pipeline] { [Pipeline] retry [Pipeline] { [Pipeline] script [Pipeline] { [Pipeline] sh git version 2.43.5 Reinitialized existing Git repository in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/.git/ .git HEAD is now at a306f4894 dm: upgrade go-mysql to v1.12.0 POST git-upload-pack (656 bytes) From https://github.com/pingcap/tiflow = [up to date] master -> origin/master = [up to date] refs/pull/12118/head -> origin/pr/12118/head Previous HEAD position was a306f4894 dm: upgrade go-mysql to v1.12.0 HEAD is now at 4c631d595 kafka(ticdc): ddl sink close the underline sink if send ddl or checkpoint failed and refactor the kafka ddl sink (#12112) ๐Ÿšง Checkouting to base SHA:4c631d5951854ac91765b729fe3849d8e8d5ed8a... HEAD is now at 4c631d595 kafka(ticdc): ddl sink close the underline sink if send ddl or checkpoint failed and refactor the kafka ddl sink (#12112) โœ… Checked. ๐ŸŽ‰ ๐Ÿงพ HEAD info: 4c631d5951854ac91765b729fe3849d8e8d5ed8a 4c631d595 kafka(ticdc): ddl sink close the underline sink if send ddl or checkpoint failed and refactor the kafka ddl sink (#12112) 3cf60079e codec(ticdc): modify DDL type support about debezium protocol (#12113) 9ed3d76c5 consumer(ticdc): adjust consumer log level (#12092) ๐Ÿšง Pre-merge heads of pull requests to base SHA: 4c631d5951854ac91765b729fe3849d8e8d5ed8a ... Updating 4c631d595..a306f4894 Fast-forward go.mod | 5 ++--- go.sum | 8 ++------ 2 files changed, 4 insertions(+), 9 deletions(-) ๐Ÿงพ Pre-merged result: a306f48949e037965a7c1963f500f7e0abbfba51 a306f4894 dm: upgrade go-mysql to v1.12.0 4c631d595 kafka(ticdc): ddl sink close the underline sink if send ddl or checkpoint failed and refactor the kafka ddl sink (#12112) 3cf60079e codec(ticdc): modify DDL type support about debezium protocol (#12113) โœ… Pre merged ๐ŸŽ‰ โœ… ~~~~~All done.~~~~~~ [Pipeline] } [Pipeline] // script [Pipeline] } [Pipeline] // retry [Pipeline] } Cache not saved (git/pingcap/tiflow/rev-4c631d5-a306f48 already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] stage [Pipeline] { (prepare) [Pipeline] timeout Timeout set to expire in 20 min [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/third_party_download [Pipeline] { [Pipeline] script [Pipeline] { [Pipeline] echo computeBranchFromPR component: tidb, prTargetBranch: master, prTitle: dm: upgrade go-mysql to v1.12.0, trunkBranch: master [Pipeline] echo computeBranchFromPR component: pd, prTargetBranch: master, prTitle: dm: upgrade go-mysql to v1.12.0, trunkBranch: master [Pipeline] echo computeBranchFromPR component: tikv, prTargetBranch: master, prTitle: dm: upgrade go-mysql to v1.12.0, trunkBranch: master [Pipeline] echo computeBranchFromPR component: tiflash, prTargetBranch: master, prTitle: dm: upgrade go-mysql to v1.12.0, trunkBranch: master [Pipeline] retry [Pipeline] { [Pipeline] sh + export TIDB_BRANCH=master + TIDB_BRANCH=master + export PD_BRANCH=master + PD_BRANCH=master + export TIKV_BRANCH=master + TIKV_BRANCH=master + export TIFLASH_BRANCH=master + TIFLASH_BRANCH=master + cd ../tiflow + ./scripts/download-integration-test-binaries.sh master Downloading binaries... >>> Downloading tidb-server.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/tidb/d9fb3a53376157be49b5a7941e6e689a33c81eb0/centos7/tidb-server.tar.gz 2025-04-04 15:18:06 URL:http://fileserver.pingcap.net/download/builds/pingcap/tidb/d9fb3a53376157be49b5a7941e6e689a33c81eb0/centos7/tidb-server.tar.gz [620275338/620275338] -> "tmp/tidb-server.tar.gz" [1] >>> Downloading pd-server.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/pd/9d6bbb8529d9d2acb9cf324a65e94ece465f6346/centos7/pd-server.tar.gz 2025-04-04 15:18:24 URL:http://fileserver.pingcap.net/download/builds/pingcap/pd/9d6bbb8529d9d2acb9cf324a65e94ece465f6346/centos7/pd-server.tar.gz [228444265/228444265] -> "tmp/pd-server.tar.gz" [1] >>> Downloading tikv-server.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/tikv/3eb3439fd88f0db9de06ef92e845e80b32c9b49a/centos7/tikv-server.tar.gz 2025-04-04 15:18:50 URL:http://fileserver.pingcap.net/download/builds/pingcap/tikv/3eb3439fd88f0db9de06ef92e845e80b32c9b49a/centos7/tikv-server.tar.gz [817400808/817400808] -> "tmp/tikv-server.tar.gz" [1] >>> Downloading tiflash.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/tiflash/master/8ecb79fcf30098e6786f4d0440ead2a879a3ad4a/centos7/tiflash.tar.gz 2025-04-04 15:19:19 URL:http://fileserver.pingcap.net/download/builds/pingcap/tiflash/master/8ecb79fcf30098e6786f4d0440ead2a879a3ad4a/centos7/tiflash.tar.gz [553235170/553235170] -> "tmp/tiflash.tar.gz" [1] >>> Downloading minio.tar.gz from http://fileserver.pingcap.net/download/minio.tar.gz 2025-04-04 15:19:25 URL:http://fileserver.pingcap.net/download/minio.tar.gz [17718777/17718777] -> "tmp/minio.tar.gz" [1] >>> Downloading etcd.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/cdc/etcd-v3.4.7-linux-amd64.tar.gz 2025-04-04 15:19:26 URL:http://fileserver.pingcap.net/download/builds/pingcap/cdc/etcd-v3.4.7-linux-amd64.tar.gz [17310840/17310840] -> "tmp/etcd.tar.gz" [1] >>> Downloading schema-registry.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/cdc/schema-registry.tar.gz 2025-04-04 15:19:38 URL:http://fileserver.pingcap.net/download/builds/pingcap/cdc/schema-registry.tar.gz [278386006/278386006] -> "tmp/schema-registry.tar.gz" [1] >>> Downloading go-ycsb from http://fileserver.pingcap.net/download/builds/pingcap/go-ycsb/test-br/go-ycsb 2025-04-04 15:19:41 URL:http://fileserver.pingcap.net/download/builds/pingcap/go-ycsb/test-br/go-ycsb [45975512/45975512] -> "third_bin/go-ycsb" [1] >>> Downloading jq from http://fileserver.pingcap.net/download/builds/pingcap/test/jq-1.6/jq-linux64 2025-04-04 15:19:41 URL:http://fileserver.pingcap.net/download/builds/pingcap/test/jq-1.6/jq-linux64 [3953824/3953824] -> "third_bin/jq" [1] Download SUCCESS + ls -alh ./bin total 2.0G drwxr-sr-x. 6 jenkins jenkins 4.0K Apr 4 15:19 . drwxr-sr-x. 19 jenkins jenkins 4.0K Apr 4 15:19 .. drwxr-sr-x. 2 jenkins jenkins 4.0K May 19 2023 bin drwxr-sr-x. 4 jenkins jenkins 4.0K May 10 2023 etc -rwxr-xr-x. 1 jenkins jenkins 17M Apr 2 2020 etcdctl -rwxr-xr-x. 1 jenkins jenkins 44M Apr 4 15:19 go-ycsb -rwxr-xr-x. 1 jenkins jenkins 3.8M Apr 4 15:19 jq drwxr-sr-x. 3 jenkins jenkins 4.0K May 10 2023 lib lrwxrwxrwx. 1 jenkins jenkins 13 Apr 3 12:33 libc++.so.1 -> libc++.so.1.0 -rwxr-xr-x. 1 jenkins jenkins 1.2M Dec 18 2023 libc++.so.1.0 lrwxrwxrwx. 1 jenkins jenkins 16 Apr 3 12:33 libc++abi.so.1 -> libc++abi.so.1.0 -rwxr-xr-x. 1 jenkins jenkins 353K Dec 18 2023 libc++abi.so.1.0 lrwxrwxrwx. 1 jenkins jenkins 13 Apr 3 12:33 libgmssl.so -> libgmssl.so.3 lrwxrwxrwx. 1 jenkins jenkins 15 Apr 3 12:33 libgmssl.so.3 -> libgmssl.so.3.0 -rwxr-xr-x. 1 jenkins jenkins 2.2M Apr 3 11:49 libgmssl.so.3.0 -rwxr-xr-x. 1 jenkins jenkins 346M Apr 3 12:34 libtiflash_proxy.so -rwxr-xr-x. 1 jenkins jenkins 50M Jul 29 2020 minio -rwxr-xr-x. 1 jenkins jenkins 41M Apr 3 15:30 pd-api-bench -rwxr-xr-x. 1 jenkins jenkins 44M Apr 3 15:29 pd-ctl -rwxr-xr-x. 1 jenkins jenkins 40M Apr 3 15:30 pd-heartbeat-bench -rwxr-xr-x. 1 jenkins jenkins 35M Apr 3 15:29 pd-recover -rwxr-xr-x. 1 jenkins jenkins 37M Apr 3 15:30 pd-region-bench -rwxr-xr-x. 1 jenkins jenkins 115M Apr 3 15:29 pd-server -rwxr-xr-x. 1 jenkins jenkins 37M Apr 3 15:30 pd-tso-bench -rwxr-xr-x. 1 jenkins jenkins 15M Apr 3 15:30 pd-ut -rwxr-xr-x. 1 jenkins jenkins 35M Apr 3 15:30 regions-dump drwxr-sr-x. 4 jenkins jenkins 4.0K May 10 2023 share -rwxr-xr-x. 1 jenkins jenkins 35M Apr 3 15:30 stores-dump -rwxr-xr-x. 1 jenkins jenkins 241M Apr 3 19:14 tidb-server -rwxr-xr-x. 1 jenkins jenkins 443M Apr 3 12:33 tiflash -rwxr-xr-x. 1 jenkins jenkins 373M Apr 2 11:56 tikv-server -rwxr-xr-x. 1 jenkins jenkins 2.3M Apr 3 15:30 xprog + make check_third_party_binary go: downloading go1.23.4 (linux/amd64) /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/tidb-server /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/tikv-server /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/pd-server /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/tiflash /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/pd-ctl /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/go-ycsb /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/etcdctl /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/jq /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/minio /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/bin/schema-registry-start + cd - /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/third_party_download + mkdir -p bin + mv ../tiflow/bin/bin ../tiflow/bin/etc ../tiflow/bin/etcdctl ../tiflow/bin/go-ycsb ../tiflow/bin/jq ../tiflow/bin/lib ../tiflow/bin/libc++.so.1 ../tiflow/bin/libc++.so.1.0 ../tiflow/bin/libc++abi.so.1 ../tiflow/bin/libc++abi.so.1.0 ../tiflow/bin/libgmssl.so ../tiflow/bin/libgmssl.so.3 ../tiflow/bin/libgmssl.so.3.0 ../tiflow/bin/libtiflash_proxy.so ../tiflow/bin/minio ../tiflow/bin/pd-api-bench ../tiflow/bin/pd-ctl ../tiflow/bin/pd-heartbeat-bench ../tiflow/bin/pd-recover ../tiflow/bin/pd-region-bench ../tiflow/bin/pd-server ../tiflow/bin/pd-tso-bench ../tiflow/bin/pd-ut ../tiflow/bin/regions-dump ../tiflow/bin/share ../tiflow/bin/stores-dump ../tiflow/bin/tidb-server ../tiflow/bin/tiflash ../tiflow/bin/tikv-server ../tiflow/bin/xprog ./bin/ + ls -alh ./bin total 2.0G drwxr-sr-x. 6 jenkins jenkins 4.0K Apr 4 15:19 . drwxr-sr-x. 3 jenkins jenkins 4.0K Apr 4 15:19 .. drwxr-sr-x. 2 jenkins jenkins 4.0K May 19 2023 bin drwxr-sr-x. 4 jenkins jenkins 4.0K May 10 2023 etc -rwxr-xr-x. 1 jenkins jenkins 17M Apr 2 2020 etcdctl -rwxr-xr-x. 1 jenkins jenkins 44M Apr 4 15:19 go-ycsb -rwxr-xr-x. 1 jenkins jenkins 3.8M Apr 4 15:19 jq drwxr-sr-x. 3 jenkins jenkins 4.0K May 10 2023 lib lrwxrwxrwx. 1 jenkins jenkins 13 Apr 3 12:33 libc++.so.1 -> libc++.so.1.0 -rwxr-xr-x. 1 jenkins jenkins 1.2M Dec 18 2023 libc++.so.1.0 lrwxrwxrwx. 1 jenkins jenkins 16 Apr 3 12:33 libc++abi.so.1 -> libc++abi.so.1.0 -rwxr-xr-x. 1 jenkins jenkins 353K Dec 18 2023 libc++abi.so.1.0 lrwxrwxrwx. 1 jenkins jenkins 13 Apr 3 12:33 libgmssl.so -> libgmssl.so.3 lrwxrwxrwx. 1 jenkins jenkins 15 Apr 3 12:33 libgmssl.so.3 -> libgmssl.so.3.0 -rwxr-xr-x. 1 jenkins jenkins 2.2M Apr 3 11:49 libgmssl.so.3.0 -rwxr-xr-x. 1 jenkins jenkins 346M Apr 3 12:34 libtiflash_proxy.so -rwxr-xr-x. 1 jenkins jenkins 50M Jul 29 2020 minio -rwxr-xr-x. 1 jenkins jenkins 41M Apr 3 15:30 pd-api-bench -rwxr-xr-x. 1 jenkins jenkins 44M Apr 3 15:29 pd-ctl -rwxr-xr-x. 1 jenkins jenkins 40M Apr 3 15:30 pd-heartbeat-bench -rwxr-xr-x. 1 jenkins jenkins 35M Apr 3 15:29 pd-recover -rwxr-xr-x. 1 jenkins jenkins 37M Apr 3 15:30 pd-region-bench -rwxr-xr-x. 1 jenkins jenkins 115M Apr 3 15:29 pd-server -rwxr-xr-x. 1 jenkins jenkins 37M Apr 3 15:30 pd-tso-bench -rwxr-xr-x. 1 jenkins jenkins 15M Apr 3 15:30 pd-ut -rwxr-xr-x. 1 jenkins jenkins 35M Apr 3 15:30 regions-dump drwxr-sr-x. 4 jenkins jenkins 4.0K May 10 2023 share -rwxr-xr-x. 1 jenkins jenkins 35M Apr 3 15:30 stores-dump -rwxr-xr-x. 1 jenkins jenkins 241M Apr 3 19:14 tidb-server -rwxr-xr-x. 1 jenkins jenkins 443M Apr 3 12:33 tiflash -rwxr-xr-x. 1 jenkins jenkins 373M Apr 2 11:56 tikv-server -rwxr-xr-x. 1 jenkins jenkins 2.3M Apr 3 15:30 xprog + ./bin/tidb-server -V Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore + ./bin/pd-server -V Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 + ./bin/tikv-server -V TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + ./bin/tiflash --version TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored [Pipeline] } [Pipeline] // retry [Pipeline] } [Pipeline] // script [Pipeline] } [Pipeline] // dir [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow [Pipeline] { [Pipeline] cache Cache not restored (no such key found) [Pipeline] { [Pipeline] sh + ls -alh ./bin total 8.0K drwxr-sr-x. 2 jenkins jenkins 4.0K Apr 4 15:19 . drwxr-sr-x. 19 jenkins jenkins 4.0K Apr 4 15:19 .. + '[' -f ./bin/cdc ']' + make cdc CGO_ENABLED=0 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v9.0.0-beta.1-10-ga306f4894" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2025-04-04 07:19:48" -X "github.com/pingcap/tiflow/pkg/version.GitHash=a306f48949e037965a7c1963f500f7e0abbfba51" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v9.0.0-beta.1-10-ga306f4894"' -o bin/cdc ./cmd/cdc go: downloading github.com/spf13/cobra v1.8.1 go: downloading github.com/pingcap/failpoint v0.0.0-20240528011301-b51a646c7c86 go: downloading google.golang.org/grpc v1.64.0 go: downloading github.com/fatih/color v1.18.0 go: downloading go.uber.org/zap v1.27.0 go: downloading github.com/coreos/go-semver v0.3.1 go: downloading github.com/tikv/pd/client v0.0.0-20241111073742-238d4d79ea31 go: downloading github.com/go-sql-driver/mysql v1.7.1 go: downloading github.com/spf13/pflag v1.0.5 go: downloading github.com/xdg/scram v1.0.5 go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20241223052309-3735ed55a394 go: downloading github.com/pingcap/log v1.1.1-0.20241212030209-7e3ff8601a2a go: downloading github.com/pingcap/kvproto v0.0.0-20241120022153-92b0414aeed8 go: downloading github.com/gin-gonic/gin v1.9.1 go: downloading github.com/IBM/sarama v1.41.2 go: downloading github.com/prometheus/client_golang v1.20.5 go: downloading github.com/pingcap/errors v0.11.5-0.20240318064555-6bd07397691f go: downloading golang.org/x/sync v0.10.0 go: downloading github.com/BurntSushi/toml v1.4.1-0.20240526193622-a339e1f7089c go: downloading golang.org/x/net v0.33.0 go: downloading github.com/tinylib/msgp v1.1.6 go: downloading github.com/dustin/go-humanize v1.0.1 go: downloading github.com/pingcap/sysutil v1.0.1-0.20240311050922-ae81ee01f3a5 go: downloading github.com/tikv/client-go/v2 v2.0.8-0.20241209094930-06d7f4b9233b go: downloading github.com/apache/pulsar-client-go v0.11.0 go: downloading go.uber.org/multierr v1.11.0 go: downloading github.com/aws/aws-sdk-go-v2 v1.19.1 go: downloading github.com/DATA-DOG/go-sqlmock v1.5.0 go: downloading github.com/imdario/mergo v0.3.16 go: downloading github.com/pingcap/tidb/pkg/parser v0.0.0-20241223052309-3735ed55a394 go: downloading github.com/google/btree v1.1.2 go: downloading github.com/r3labs/diff v1.1.0 go: downloading github.com/google/uuid v1.6.0 go: downloading go.etcd.io/etcd/client/v3 v3.5.12 go: downloading golang.org/x/time v0.7.0 go: downloading go.etcd.io/etcd/server/v3 v3.5.12 go: downloading go.uber.org/atomic v1.11.0 go: downloading github.com/cockroachdb/pebble v1.1.0 go: downloading github.com/swaggo/files v0.0.0-20210815190702-a29dd2bc99b2 go: downloading github.com/swaggo/gin-swagger v1.2.0 go: downloading cloud.google.com/go/storage v1.39.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v1.16.0 go: downloading github.com/aws/aws-sdk-go v1.55.5 go: downloading github.com/KimMachineGun/automemlimit v0.2.4 go: downloading github.com/json-iterator/go v1.1.12 go: downloading github.com/shirou/gopsutil/v3 v3.24.5 go: downloading github.com/gogo/protobuf v1.3.2 go: downloading github.com/soheilhy/cmux v0.1.5 go: downloading github.com/tikv/pd v1.1.0-beta.0.20240407022249-7179657d129b go: downloading github.com/goccy/go-json v0.10.2 go: downloading github.com/stretchr/testify v1.10.0 go: downloading go.etcd.io/etcd/api/v3 v3.5.12 go: downloading golang.org/x/sys v0.28.0 go: downloading github.com/benbjohnson/clock v1.1.0 go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.12 go: downloading github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 go: downloading cloud.google.com/go v0.112.2 go: downloading github.com/cenkalti/backoff/v4 v4.2.1 go: downloading github.com/gavv/monotime v0.0.0-20190418164738-30dba4353424 go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 go: downloading github.com/klauspost/compress v1.17.9 go: downloading github.com/pierrec/lz4/v4 v4.1.18 go: downloading github.com/robfig/cron v1.2.0 go: downloading github.com/hashicorp/golang-lru v0.5.1 go: downloading github.com/uber-go/atomic v1.4.0 go: downloading go.etcd.io/etcd/pkg/v3 v3.5.12 go: downloading github.com/jcmturner/gokrb5/v8 v8.4.4 go: downloading github.com/segmentio/kafka-go v0.4.41-0.20230526171612-f057b1d369cd go: downloading github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 go: downloading golang.org/x/oauth2 v0.24.0 go: downloading github.com/go-mysql-org/go-mysql v1.12.0 go: downloading github.com/pingcap/check v0.0.0-20211026125417-57bd13f7b5f0 go: downloading github.com/mattn/go-colorable v0.1.13 go: downloading github.com/mattn/go-isatty v0.0.20 go: downloading golang.org/x/term v0.27.0 go: downloading github.com/pingcap/tidb-dashboard v0.0.0-20240326110213-9768844ff5d7 go: downloading github.com/golang/mock v1.6.0 go: downloading gopkg.in/natefinch/lumberjack.v2 v2.2.1 go: downloading github.com/swaggo/swag v1.16.3 go: downloading github.com/xdg/stringprep v1.0.3 go: downloading golang.org/x/crypto v0.31.0 go: downloading github.com/containerd/cgroups v1.0.4 go: downloading github.com/philhofer/fwd v1.1.1 go: downloading github.com/gin-contrib/sse v0.1.0 go: downloading github.com/go-playground/validator/v10 v10.14.0 go: downloading github.com/pelletier/go-toml/v2 v2.0.8 go: downloading github.com/ugorji/go/codec v1.2.11 go: downloading google.golang.org/protobuf v1.35.2 go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd go: downloading github.com/golang/protobuf v1.5.4 go: downloading google.golang.org/genproto/googleapis/api v0.0.0-20240401170217-c3f982113cda go: downloading github.com/aws/smithy-go v1.13.5 go: downloading google.golang.org/genproto v0.0.0-20240401170217-c3f982113cda go: downloading github.com/beorn7/perks v1.0.1 go: downloading github.com/cespare/xxhash/v2 v2.3.0 go: downloading github.com/prometheus/client_model v0.6.1 go: downloading github.com/prometheus/common v0.61.0 go: downloading github.com/prometheus/procfs v0.15.1 go: downloading github.com/coreos/go-systemd/v22 v22.5.0 go: downloading github.com/bits-and-blooms/bitset v1.14.3 go: downloading github.com/linkedin/goavro/v2 v2.11.1 go: downloading github.com/pkg/errors v0.9.1 go: downloading github.com/sirupsen/logrus v1.9.3 go: downloading github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc go: downloading github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 go: downloading github.com/eapache/go-resiliency v1.4.0 go: downloading github.com/eapache/queue v1.1.0 go: downloading github.com/hashicorp/go-multierror v1.1.1 go: downloading github.com/jcmturner/gofork v1.7.6 go: downloading github.com/opentracing/opentracing-go v1.2.0 go: downloading github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a go: downloading github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 go: downloading github.com/stretchr/objx v0.5.2 go: downloading cloud.google.com/go/compute/metadata v0.3.0 go: downloading cloud.google.com/go/iam v1.1.7 go: downloading github.com/googleapis/gax-go/v2 v2.12.3 go: downloading google.golang.org/api v0.170.0 go: downloading github.com/AthenZ/athenz v1.10.39 go: downloading github.com/pierrec/lz4 v2.6.1+incompatible go: downloading github.com/grpc-ecosystem/grpc-gateway v1.16.0 go: downloading go.etcd.io/bbolt v1.3.9 go: downloading github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75 go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 go: downloading go.opentelemetry.io/otel/sdk v1.24.0 go: downloading go.opentelemetry.io/otel v1.24.0 go: downloading go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.22.0 go: downloading sigs.k8s.io/yaml v1.4.0 go: downloading github.com/jcmturner/dnsutils/v2 v2.0.0 go: downloading github.com/hashicorp/go-uuid v1.0.3 go: downloading github.com/tklauser/go-sysconf v0.3.12 go: downloading github.com/xdg-go/scram v1.1.2 go: downloading github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5 go: downloading github.com/docker/go-units v0.5.0 go: downloading github.com/Masterminds/semver v1.5.0 go: downloading filippo.io/edwards25519 v1.1.0 go: downloading google.golang.org/genproto/googleapis/rpc v0.0.0-20240515191416-fc5f0ca64291 go: downloading golang.org/x/text v0.21.0 go: downloading github.com/godbus/dbus/v5 v5.0.4 go: downloading github.com/opencontainers/runtime-spec v1.0.2 go: downloading github.com/cilium/ebpf v0.4.0 go: downloading golang.org/x/tools v0.28.0 go: downloading github.com/go-openapi/spec v0.21.0 go: downloading github.com/gabriel-vasile/mimetype v1.4.2 go: downloading github.com/go-playground/universal-translator v0.18.1 go: downloading github.com/leodido/go-urn v1.2.4 go: downloading github.com/KyleBanks/depth v1.2.1 go: downloading github.com/joomcode/errorx v1.0.1 go: downloading gorm.io/gorm v1.25.11 go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 go: downloading github.com/cockroachdb/errors v1.11.1 go: downloading github.com/cockroachdb/redact v1.1.5 go: downloading github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 go: downloading golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 go: downloading github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 go: downloading github.com/spaolacci/murmur3 v1.1.0 go: downloading golang.org/x/mod v0.22.0 go: downloading github.com/golang/snappy v0.0.4 go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 go: downloading github.com/hashicorp/errwrap v1.0.0 go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading github.com/twmb/murmur3 v1.1.6 go: downloading github.com/dgryski/go-farm v0.0.0-20240924180020-3414d57e47da go: downloading github.com/cloudfoundry/gosigar v1.3.6 go: downloading go.opencensus.io v0.23.1-0.20220331163232-052120675fac go: downloading go.opentelemetry.io/otel/trace v1.24.0 go: downloading go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.22.0 go: downloading github.com/aws/aws-sdk-go-v2/config v1.18.30 go: downloading github.com/aws/aws-sdk-go-v2/credentials v1.13.29 go: downloading github.com/aws/aws-sdk-go-v2/service/glue v1.58.1 go: downloading github.com/jarcoal/httpmock v1.2.0 go: downloading github.com/mailru/easyjson v0.7.7 go: downloading github.com/golang-jwt/jwt v3.2.2+incompatible go: downloading github.com/99designs/keyring v1.2.1 go: downloading github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 go: downloading go.etcd.io/etcd/raft/v3 v3.5.12 go: downloading github.com/xiang90/probing v0.0.0-20221125231312-a49e3df8f510 go: downloading github.com/jonboulle/clockwork v0.4.0 go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 go: downloading github.com/go-resty/resty/v2 v2.11.0 go: downloading github.com/ks3sdklib/aws-sdk-go v1.2.9 go: downloading github.com/coocood/freecache v1.2.1 go: downloading github.com/pingcap/tipb v0.0.0-20241105053214-f91fdb81a69e go: downloading github.com/jcmturner/aescts/v2 v2.0.0 go: downloading github.com/jcmturner/rpc/v2 v2.0.3 go: downloading github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 go: downloading github.com/otiai10/copy v1.2.0 go: downloading github.com/gorilla/mux v1.8.1 go: downloading github.com/blacktear23/go-proxyprotocol v1.0.6 go: downloading github.com/pingcap/fn v1.0.0 go: downloading github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 go: downloading github.com/shopspring/decimal v1.3.0 go: downloading github.com/tidwall/btree v1.7.0 go: downloading github.com/qri-io/jsonschema v0.2.1 go: downloading github.com/tiancaiamao/appdash v0.0.0-20181126055449-889f96f722a2 go: downloading github.com/sourcegraph/appdash-data v0.0.0-20151005221446-73f23eafcf67 go: downloading github.com/edwingeng/deque v0.0.0-20191220032131-8596380dee17 go: downloading github.com/gorilla/websocket v1.5.1 go: downloading go.opentelemetry.io/otel/metric v1.24.0 go: downloading github.com/xdg-go/pbkdf2 v1.0.0 go: downloading github.com/xdg-go/stringprep v1.0.4 go: downloading github.com/go-playground/locales v0.14.1 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec go: downloading github.com/tklauser/numcpus v0.6.1 go: downloading github.com/carlmjohnson/flagext v0.21.0 go: downloading github.com/spkg/bom v1.0.0 go: downloading github.com/xitongsys/parquet-go v1.6.3-0.20240520233950-75e935fc3e17 go: downloading github.com/ardielle/ardielle-go v1.5.2 go: downloading github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.6 go: downloading github.com/aws/aws-sdk-go-v2/service/sso v1.12.14 go: downloading github.com/aws/aws-sdk-go-v2/service/sts v1.20.1 go: downloading github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.14 go: downloading github.com/golang-jwt/jwt/v4 v4.5.1 go: downloading github.com/aws/aws-sdk-go-v2/internal/ini v1.3.37 go: downloading go.etcd.io/etcd/client/v2 v2.305.12 go: downloading github.com/dvsekhvalnov/jose2go v1.5.0 go: downloading github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c go: downloading github.com/mtibben/percent v0.2.1 go: downloading github.com/jinzhu/now v1.1.5 go: downloading github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da go: downloading github.com/go-openapi/jsonreference v0.21.0 go: downloading github.com/go-openapi/jsonpointer v0.21.0 go: downloading github.com/go-openapi/swag v0.23.0 go: downloading github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b go: downloading github.com/getsentry/sentry-go v0.27.0 go: downloading gopkg.in/yaml.v2 v2.4.0 go: downloading go.uber.org/mock v0.4.0 go: downloading github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef go: downloading github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.36 go: downloading github.com/jfcg/sorty/v2 v2.1.0 go: downloading github.com/dgraph-io/ristretto v0.1.1 go: downloading github.com/dolthub/swiss v0.2.1 go: downloading github.com/opentracing/basictracer-go v1.1.0 go: downloading github.com/cheggaaa/pb/v3 v3.0.8 go: downloading github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible go: downloading github.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0 go: downloading github.com/danjacques/gofslock v0.0.0-20240212154529-d899e02bfe22 go: downloading github.com/influxdata/tdigest v0.0.1 go: downloading github.com/yangkeao/ldap/v3 v3.4.5-0.20230421065457-369a3bab1117 go: downloading github.com/lestrrat-go/jwx/v2 v2.0.21 go: downloading github.com/jellydator/ttlcache/v3 v3.0.1 go: downloading github.com/google/pprof v0.0.0-20241001023024-f4c0cfd0cf1d go: downloading github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.30 go: downloading go.opentelemetry.io/proto/otlp v1.1.0 go: downloading github.com/jinzhu/inflection v1.0.0 go: downloading github.com/go-logr/logr v1.4.1 go: downloading github.com/kr/pretty v0.3.1 go: downloading github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df go: downloading github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.30 go: downloading github.com/pingcap/badger v1.5.1-0.20241015064302-38533b6cbf8d go: downloading github.com/jedib0t/go-pretty/v6 v6.2.2 go: downloading github.com/vbauerster/mpb/v7 v7.5.3 go: downloading github.com/dolthub/maphash v0.1.0 go: downloading github.com/jfcg/sixb v1.3.8 go: downloading github.com/qri-io/jsonpointer v0.1.1 go: downloading github.com/apache/thrift v0.16.0 go: downloading github.com/robfig/cron/v3 v3.0.1 go: downloading github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f go: downloading github.com/VividCortex/ewma v1.2.0 go: downloading github.com/mattn/go-runewidth v0.0.16 go: downloading github.com/go-logr/stdr v1.2.2 go: downloading github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 go: downloading github.com/go-asn1-ber/asn1-ber v1.5.4 go: downloading github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1 go: downloading github.com/kr/text v0.2.0 go: downloading github.com/rogpeppe/go-internal v1.13.1 go: downloading github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c go: downloading github.com/kylelemons/godebug v1.1.0 go: downloading k8s.io/api v0.29.11 go: downloading github.com/emirpasic/gods v1.18.1 go: downloading github.com/josharian/intern v1.0.0 go: downloading github.com/apache/arrow/go/v12 v12.0.1 go: downloading github.com/goccy/go-reflect v1.2.0 go: downloading github.com/rivo/uniseg v0.4.7 go: downloading github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d go: downloading github.com/golang-jwt/jwt/v5 v5.2.1 go: downloading github.com/golang/glog v1.2.0 go: downloading cloud.google.com/go/kms v1.15.8 go: downloading github.com/ncw/directio v1.0.5 go: downloading github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 go: downloading github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 go: downloading github.com/klauspost/cpuid v1.3.1 go: downloading github.com/lestrrat-go/httprc v1.0.5 go: downloading github.com/lestrrat-go/iter v1.0.2 go: downloading github.com/lestrrat-go/blackmagic v1.0.2 go: downloading github.com/lestrrat-go/option v1.0.1 go: downloading github.com/lestrrat-go/httpcc v1.0.1 go: downloading golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 go: downloading github.com/google/flatbuffers v2.0.8+incompatible go: downloading github.com/zeebo/xxh3 v1.0.2 go: downloading github.com/andybalholm/brotli v1.0.5 go: downloading github.com/klauspost/cpuid/v2 v2.2.4 go: downloading k8s.io/apimachinery v0.29.11 go: downloading gopkg.in/inf.v0 v0.9.1 go: downloading github.com/google/gofuzz v1.2.0 go: downloading sigs.k8s.io/structured-merge-diff/v4 v4.4.1 go: downloading sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd go: downloading k8s.io/klog/v2 v2.120.1 go: downloading k8s.io/utils v0.0.0-20230726121419-3b25d923346b go: downloading github.com/jmespath/go-jmespath v0.4.0 go: downloading go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 go: downloading github.com/google/s2a-go v0.1.7 go: downloading github.com/googleapis/enterprise-certificate-proxy v0.3.2 go: downloading github.com/felixge/httpsnoop v1.0.4 + '[' -f ./bin/cdc_pulsar_consumer ']' + make pulsar_consumer CGO_ENABLED=0 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v9.0.0-beta.1-10-ga306f4894" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2025-04-04 07:21:53" -X "github.com/pingcap/tiflow/pkg/version.GitHash=a306f48949e037965a7c1963f500f7e0abbfba51" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v9.0.0-beta.1-10-ga306f4894"' -o bin/cdc_pulsar_consumer ./cmd/pulsar-consumer/main.go + '[' -f ./bin/cdc.test ']' + make integration_test_build cd tools/check && GO111MODULE=on go build -mod=mod -o ../bin/failpoint-ctl github.com/pingcap/failpoint/failpoint-ctl go: downloading github.com/pingcap/failpoint v0.0.0-20210316064728-7acb0f0a3dfd go: downloading github.com/sergi/go-diff v1.1.0 CGO_ENABLED=0 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v9.0.0-beta.1-10-ga306f4894" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2025-04-04 07:22:01" -X "github.com/pingcap/tiflow/pkg/version.GitHash=a306f48949e037965a7c1963f500f7e0abbfba51" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v9.0.0-beta.1-10-ga306f4894"' -o bin/cdc_storage_consumer ./cmd/storage-consumer/main.go CGO_ENABLED=1 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v9.0.0-beta.1-10-ga306f4894" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2025-04-04 07:22:01" -X "github.com/pingcap/tiflow/pkg/version.GitHash=a306f48949e037965a7c1963f500f7e0abbfba51" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v9.0.0-beta.1-10-ga306f4894"' -o bin/cdc_kafka_consumer ./cmd/kafka-consumer go: downloading github.com/confluentinc/confluent-kafka-go/v2 v2.3.0 go: downloading github.com/DataDog/zstd v1.5.5 CGO_ENABLED=0 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v9.0.0-beta.1-10-ga306f4894" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2025-04-04 07:22:01" -X "github.com/pingcap/tiflow/pkg/version.GitHash=a306f48949e037965a7c1963f500f7e0abbfba51" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v9.0.0-beta.1-10-ga306f4894"' -o bin/cdc_pulsar_consumer ./cmd/pulsar-consumer/main.go CGO_ENABLED=0 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v9.0.0-beta.1-10-ga306f4894" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2025-04-04 07:22:01" -X "github.com/pingcap/tiflow/pkg/version.GitHash=a306f48949e037965a7c1963f500f7e0abbfba51" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v9.0.0-beta.1-10-ga306f4894"' -o bin/oauth2-server ./cmd/oauth2-server/main.go go: downloading github.com/go-oauth2/oauth2/v4 v4.5.2 go: downloading github.com/tidwall/buntdb v1.3.0 go: downloading github.com/tidwall/rtred v0.1.2 go: downloading github.com/tidwall/grect v0.1.4 go: downloading github.com/tidwall/match v1.1.1 go: downloading github.com/tidwall/gjson v1.14.3 go: downloading github.com/tidwall/tinyqueue v0.1.1 go: downloading github.com/tidwall/pretty v1.2.0 CGO_ENABLED=0 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v9.0.0-beta.1-10-ga306f4894" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2025-04-04 07:22:01" -X "github.com/pingcap/tiflow/pkg/version.GitHash=a306f48949e037965a7c1963f500f7e0abbfba51" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v9.0.0-beta.1-10-ga306f4894"' -o bin/sync_diff_inspector ./sync_diff_inspector go: downloading github.com/olekukonko/tablewriter v0.0.5 $(echo $(for p in $(go list ./... | grep -vE 'vendor|proto|tiflow/tests|integration|testing_utils|pb|pbmock|tiflow/bin'); do echo ${p#"github.com/pingcap/tiflow/"}|grep -v "github.com/pingcap/tiflow"; done) | xargs tools/bin/failpoint-ctl enable >/dev/null) go: downloading github.com/PingCAP-QE/go-sqlsmith v0.0.0-20241224111350-ad2e4f976c7c go: downloading github.com/chzyer/readline v1.5.1 go: downloading github.com/deepmap/oapi-codegen v1.9.0 go: downloading github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 go: downloading github.com/getkin/kin-openapi v0.80.0 go: downloading github.com/gogo/gateway v1.1.0 go: downloading github.com/syndtr/goleveldb v1.0.1-0.20210305035536-64b5b1c73954 go: downloading go.uber.org/dig v1.13.0 go: downloading github.com/shurcooL/httpgzip v0.0.0-20190720172056-320755c1c1b0 go: downloading github.com/mattn/go-shellwords v1.0.12 go: downloading github.com/glebarez/go-sqlite v1.21.2 go: downloading github.com/glebarez/sqlite v1.7.0 go: downloading github.com/go-ozzo/ozzo-validation/v4 v4.3.0 go: downloading go.uber.org/ratelimit v0.2.0 go: downloading gorm.io/driver/mysql v1.5.7 go: downloading github.com/VividCortex/mysqlerr v1.0.0 go: downloading go.uber.org/goleak v1.3.0 go: downloading github.com/ngaut/log v0.0.0-20210830112240-0124ec040aeb go: downloading github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 go: downloading modernc.org/libc v1.37.1 go: downloading modernc.org/sqlite v1.27.0 go: downloading github.com/bradleyjkemp/grpc-tools v0.2.5 go: downloading github.com/integralist/go-findroot v0.0.0-20160518114804-ac90681525dc go: downloading github.com/jmoiron/sqlx v1.3.3 go: downloading upper.io/db.v3 v3.7.1+incompatible go: downloading github.com/ghodss/yaml v1.0.0 go: downloading github.com/improbable-eng/grpc-web v0.12.0 go: downloading github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f go: downloading github.com/rs/cors v1.7.0 go: downloading modernc.org/memory v1.7.2 go: downloading modernc.org/mathutil v1.6.0 CGO_ENABLED=1 GO111MODULE=on go test -p 3 --race --tags=intest -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v9.0.0-beta.1-10-ga306f4894" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2025-04-04 07:22:01" -X "github.com/pingcap/tiflow/pkg/version.GitHash=a306f48949e037965a7c1963f500f7e0abbfba51" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v9.0.0-beta.1-10-ga306f4894"' -c -cover -covermode=atomic \ -coverpkg=github.com/pingcap/tiflow/... \ -o bin/cdc.test github.com/pingcap/tiflow/cmd/cdc \ || { $(echo $(for p in $(go list ./... | grep -vE 'vendor|proto|tiflow/tests|integration|testing_utils|pb|pbmock|tiflow/bin'); do echo ${p#"github.com/pingcap/tiflow/"}|grep -v "github.com/pingcap/tiflow"; done) | xargs tools/bin/failpoint-ctl disable >/dev/null); exit 1; } CGO_ENABLED=0 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v9.0.0-beta.1-10-ga306f4894" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2025-04-04 07:22:01" -X "github.com/pingcap/tiflow/pkg/version.GitHash=a306f48949e037965a7c1963f500f7e0abbfba51" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v9.0.0-beta.1-10-ga306f4894"' -o bin/cdc ./cmd/cdc/main.go \ || { $(echo $(for p in $(go list ./... | grep -vE 'vendor|proto|tiflow/tests|integration|testing_utils|pb|pbmock|tiflow/bin'); do echo ${p#"github.com/pingcap/tiflow/"}|grep -v "github.com/pingcap/tiflow"; done) | xargs tools/bin/failpoint-ctl disable >/dev/null); exit 1; } $(echo $(for p in $(go list ./... | grep -vE 'vendor|proto|tiflow/tests|integration|testing_utils|pb|pbmock|tiflow/bin'); do echo ${p#"github.com/pingcap/tiflow/"}|grep -v "github.com/pingcap/tiflow"; done) | xargs tools/bin/failpoint-ctl disable >/dev/null) + ls -alh ./bin total 1.4G drwxr-sr-x. 2 jenkins jenkins 4.0K Apr 4 15:28 . drwxr-sr-x. 19 jenkins jenkins 4.0K Apr 4 15:19 .. -rwxr-xr-x. 1 jenkins jenkins 231M Apr 4 15:28 cdc -rwxr-xr-x. 1 jenkins jenkins 364M Apr 4 15:28 cdc.test -rwxr-xr-x. 1 jenkins jenkins 269M Apr 4 15:24 cdc_kafka_consumer -rwxr-xr-x. 1 jenkins jenkins 200M Apr 4 15:24 cdc_pulsar_consumer -rwxr-xr-x. 1 jenkins jenkins 199M Apr 4 15:22 cdc_storage_consumer -rwxr-xr-x. 1 jenkins jenkins 13M Apr 4 15:24 oauth2-server -rwxr-xr-x. 1 jenkins jenkins 145M Apr 4 15:24 sync_diff_inspector + ./bin/cdc version Release Version: v9.0.0-beta.1-10-ga306f4894 Git Commit Hash: a306f48949e037965a7c1963f500f7e0abbfba51 Git Branch: HEAD UTC Build Time: 2025-04-04 07:22:01 Go Version: go1.23.4 Failpoint Build: true [Pipeline] } Cache saved successfully (binary/pingcap/tiflow/cdc-integration-pulsar-test/rev-4c631d5-a306f48) 1485649920 bytes in 24.10 secs (61634277 bytes/sec) [Pipeline] // cache [Pipeline] cache Cache not restored (no such key found) [Pipeline] { [Pipeline] sh + cp -r ../third_party_download/bin/bin ../third_party_download/bin/etc ../third_party_download/bin/etcdctl ../third_party_download/bin/go-ycsb ../third_party_download/bin/jq ../third_party_download/bin/lib ../third_party_download/bin/libc++.so.1 ../third_party_download/bin/libc++.so.1.0 ../third_party_download/bin/libc++abi.so.1 ../third_party_download/bin/libc++abi.so.1.0 ../third_party_download/bin/libgmssl.so ../third_party_download/bin/libgmssl.so.3 ../third_party_download/bin/libgmssl.so.3.0 ../third_party_download/bin/libtiflash_proxy.so ../third_party_download/bin/minio ../third_party_download/bin/pd-api-bench ../third_party_download/bin/pd-ctl ../third_party_download/bin/pd-heartbeat-bench ../third_party_download/bin/pd-recover ../third_party_download/bin/pd-region-bench ../third_party_download/bin/pd-server ../third_party_download/bin/pd-tso-bench ../third_party_download/bin/pd-ut ../third_party_download/bin/regions-dump ../third_party_download/bin/share ../third_party_download/bin/stores-dump ../third_party_download/bin/tidb-server ../third_party_download/bin/tiflash ../third_party_download/bin/tikv-server ../third_party_download/bin/xprog ./bin/ + ls -alh ./bin total 3.3G drwxr-sr-x. 6 jenkins jenkins 4.0K Apr 4 15:29 . drwxr-sr-x. 19 jenkins jenkins 4.0K Apr 4 15:19 .. drwxr-sr-x. 2 jenkins jenkins 4.0K Apr 4 15:29 bin -rwxr-xr-x. 1 jenkins jenkins 231M Apr 4 15:28 cdc -rwxr-xr-x. 1 jenkins jenkins 364M Apr 4 15:28 cdc.test -rwxr-xr-x. 1 jenkins jenkins 269M Apr 4 15:24 cdc_kafka_consumer -rwxr-xr-x. 1 jenkins jenkins 200M Apr 4 15:24 cdc_pulsar_consumer -rwxr-xr-x. 1 jenkins jenkins 199M Apr 4 15:22 cdc_storage_consumer drwxr-sr-x. 4 jenkins jenkins 4.0K Apr 4 15:29 etc -rwxr-xr-x. 1 jenkins jenkins 17M Apr 4 15:29 etcdctl -rwxr-xr-x. 1 jenkins jenkins 44M Apr 4 15:29 go-ycsb -rwxr-xr-x. 1 jenkins jenkins 3.8M Apr 4 15:29 jq drwxr-sr-x. 3 jenkins jenkins 4.0K Apr 4 15:29 lib lrwxrwxrwx. 1 jenkins jenkins 13 Apr 4 15:29 libc++.so.1 -> libc++.so.1.0 -rwxr-xr-x. 1 jenkins jenkins 1.2M Apr 4 15:29 libc++.so.1.0 lrwxrwxrwx. 1 jenkins jenkins 16 Apr 4 15:29 libc++abi.so.1 -> libc++abi.so.1.0 -rwxr-xr-x. 1 jenkins jenkins 353K Apr 4 15:29 libc++abi.so.1.0 lrwxrwxrwx. 1 jenkins jenkins 13 Apr 4 15:29 libgmssl.so -> libgmssl.so.3 lrwxrwxrwx. 1 jenkins jenkins 15 Apr 4 15:29 libgmssl.so.3 -> libgmssl.so.3.0 -rwxr-xr-x. 1 jenkins jenkins 2.2M Apr 4 15:29 libgmssl.so.3.0 -rwxr-xr-x. 1 jenkins jenkins 346M Apr 4 15:29 libtiflash_proxy.so -rwxr-xr-x. 1 jenkins jenkins 50M Apr 4 15:29 minio -rwxr-xr-x. 1 jenkins jenkins 13M Apr 4 15:24 oauth2-server -rwxr-xr-x. 1 jenkins jenkins 41M Apr 4 15:29 pd-api-bench -rwxr-xr-x. 1 jenkins jenkins 44M Apr 4 15:29 pd-ctl -rwxr-xr-x. 1 jenkins jenkins 40M Apr 4 15:29 pd-heartbeat-bench -rwxr-xr-x. 1 jenkins jenkins 35M Apr 4 15:29 pd-recover -rwxr-xr-x. 1 jenkins jenkins 37M Apr 4 15:29 pd-region-bench -rwxr-xr-x. 1 jenkins jenkins 115M Apr 4 15:29 pd-server -rwxr-xr-x. 1 jenkins jenkins 37M Apr 4 15:29 pd-tso-bench -rwxr-xr-x. 1 jenkins jenkins 15M Apr 4 15:29 pd-ut -rwxr-xr-x. 1 jenkins jenkins 35M Apr 4 15:29 regions-dump drwxr-sr-x. 4 jenkins jenkins 4.0K Apr 4 15:29 share -rwxr-xr-x. 1 jenkins jenkins 35M Apr 4 15:29 stores-dump -rwxr-xr-x. 1 jenkins jenkins 145M Apr 4 15:24 sync_diff_inspector -rwxr-xr-x. 1 jenkins jenkins 241M Apr 4 15:29 tidb-server -rwxr-xr-x. 1 jenkins jenkins 443M Apr 4 15:29 tiflash -rwxr-xr-x. 1 jenkins jenkins 373M Apr 4 15:29 tikv-server -rwxr-xr-x. 1 jenkins jenkins 2.3M Apr 4 15:29 xprog [Pipeline] } Cache saved successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-3484/tiflow-cdc) 4030750720 bytes in 64.46 secs (62531349 bytes/sec) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] stage [Pipeline] { (Tests) [Pipeline] parallel [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G00') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G01') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G02') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G03') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G04') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G05') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G06') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G07') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G08') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G09') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G10') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G11') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G12') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G13') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G14') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G15') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G16') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G17') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G00') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G01') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G02') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G03') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G04') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G05') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G06') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G07') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G08') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G09') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G10') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G11') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G12') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G13') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G14') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G15') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G16') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G17') [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_pulsar_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_pulsar_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-34gv-66s3n Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_pulsar_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-0lxq-6vnt9 Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_pulsar_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-km3g-0ld68 Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_pulsar_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-gr29-fxw7k Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_pulsar_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Agent pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-34gv-66s3n is provisioned from template pingcap_tiflow_pull_cdc_integration_pulsar_test_3484-34gvt-wc697 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/3484/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/3484/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "07d7b142946eb54844c3d472b641b04ad012da7f" jenkins/label: "pingcap_tiflow_pull_cdc_integration_pulsar_test_3484-34gvt" name: "pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-34gv-66s3n" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/rocky8_golang-1.23:latest" name: "golang" resources: limits: memory: "32Gi" cpu: "12" requests: memory: "32Gi" cpu: "12" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-34gv-66s3n" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-34gv-66s3n" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-34gv-66s3n in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test [Pipeline] { Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-2bnf-6v9zm [Pipeline] checkout The recommended git tool is: git Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_pulsar_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Agent pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-0lxq-6vnt9 is provisioned from template pingcap_tiflow_pull_cdc_integration_pulsar_test_3484-0lxqm-zwb1m --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/3484/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/3484/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "60a61fe44638bd37a1c5046f7f0fe12abf4c79cf" jenkins/label: "pingcap_tiflow_pull_cdc_integration_pulsar_test_3484-0lxqm" name: "pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-0lxq-6vnt9" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/rocky8_golang-1.23:latest" name: "golang" resources: limits: memory: "32Gi" cpu: "12" requests: memory: "32Gi" cpu: "12" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-0lxq-6vnt9" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-0lxq-6vnt9" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-0lxq-6vnt9 in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test [Pipeline] { Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-sfgg-lkh0k [Pipeline] checkout The recommended git tool is: git No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@5c7d80d1; decorates RemoteLauncher[hudson.remoting.Channel@54efe468:JNLP4-connect connection from 10.233.67.163/10.233.67.163:56884] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_pulsar_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-4ttz-989qr No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@1643a67e; decorates RemoteLauncher[hudson.remoting.Channel@553eee68:JNLP4-connect connection from 10.233.70.218/10.233.70.218:48186] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Agent pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-gr29-fxw7k is provisioned from template pingcap_tiflow_pull_cdc_integration_pulsar_test_3484-gr29d-c2dsk --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/3484/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/3484/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "4201a15d59b5034c5a6bdac9562f435b53a09fa5" jenkins/label: "pingcap_tiflow_pull_cdc_integration_pulsar_test_3484-gr29d" name: "pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-gr29-fxw7k" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/rocky8_golang-1.23:latest" name: "golang" resources: limits: memory: "32Gi" cpu: "12" requests: memory: "32Gi" cpu: "12" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-gr29-fxw7k" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-gr29-fxw7k" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_pulsar_test.yaml from git https://github.com/PingCAP-QE/ci.git Running on pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-gr29-fxw7k in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test [Pipeline] podTemplate [Pipeline] { [Pipeline] { [Pipeline] node [Pipeline] checkout Agent pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-km3g-0ld68 is provisioned from template pingcap_tiflow_pull_cdc_integration_pulsar_test_3484-km3gp-qn678 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/3484/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/3484/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "4526aeb0a14d6af8feaf098b7e9d244ecd3d1241" jenkins/label: "pingcap_tiflow_pull_cdc_integration_pulsar_test_3484-km3gp" name: "pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-km3g-0ld68" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/rocky8_golang-1.23:latest" name: "golang" resources: limits: memory: "32Gi" cpu: "12" requests: memory: "32Gi" cpu: "12" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-km3g-0ld68" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-km3g-0ld68" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-km3g-0ld68 in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test [Pipeline] { The recommended git tool is: git [Pipeline] checkout Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-jd09-mfgkq The recommended git tool is: git Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_pulsar_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-9bld-w6hq9 Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-qg8g-pl0pp > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@6b867f5f; decorates RemoteLauncher[hudson.remoting.Channel@14982fc8:JNLP4-connect connection from 10.233.107.107/10.233.107.107:58518] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Avoid second fetch Checking out Revision b2a59508c11c86ba701c4af25fc536a9475b82c6 (origin/main) No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@4fc0c8e9; decorates RemoteLauncher[hudson.remoting.Channel@5021c10b:JNLP4-connect connection from 10.233.68.63/10.233.68.63:59472] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Commit message: "feat(prow-jobs/pingcap-qe/ci): add periodic job to crawl pull requests and issues (#3449)" Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 [Pipeline] withEnv [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] stage [Pipeline] { (Test) [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow [Pipeline] { Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_pulsar_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] cache Agent pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-2bnf-6v9zm is provisioned from template pingcap_tiflow_pull_cdc_integration_pulsar_test_3484-2bnfb-146f0 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/3484/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/3484/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "1d8a3200b6f4baa94cd7a04e14d0549de4d48f42" jenkins/label: "pingcap_tiflow_pull_cdc_integration_pulsar_test_3484-2bnfb" name: "pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-2bnf-6v9zm" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/rocky8_golang-1.23:latest" name: "golang" resources: limits: memory: "32Gi" cpu: "12" requests: memory: "32Gi" cpu: "12" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-2bnf-6v9zm" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-2bnf-6v9zm" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-2bnf-6v9zm in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 Avoid second fetch Checking out Revision b2a59508c11c86ba701c4af25fc536a9475b82c6 (origin/main) Agent pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-4ttz-989qr is provisioned from template pingcap_tiflow_pull_cdc_integration_pulsar_test_3484-4ttz8-9pvzt --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/3484/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/3484/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "254c7fcb5e36a6cb80523d5426f2f16d618926cb" jenkins/label: "pingcap_tiflow_pull_cdc_integration_pulsar_test_3484-4ttz8" name: "pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-4ttz-989qr" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/rocky8_golang-1.23:latest" name: "golang" resources: limits: memory: "32Gi" cpu: "12" requests: memory: "32Gi" cpu: "12" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-4ttz-989qr" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-4ttz-989qr" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-4ttz-989qr in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f b2a59508c11c86ba701c4af25fc536a9475b82c6 # timeout=10 Commit message: "feat(prow-jobs/pingcap-qe/ci): add periodic job to crawl pull requests and issues (#3449)" Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_pulsar_test.yaml from git https://github.com/PingCAP-QE/ci.git Agent pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-sfgg-lkh0k is provisioned from template pingcap_tiflow_pull_cdc_integration_pulsar_test_3484-sfgg7-9fm67 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/3484/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/3484/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "a5d77b8077bc00e3d19d211ab036cd9fa4720de0" jenkins/label: "pingcap_tiflow_pull_cdc_integration_pulsar_test_3484-sfgg7" name: "pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-sfgg-lkh0k" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/rocky8_golang-1.23:latest" name: "golang" resources: limits: memory: "32Gi" cpu: "12" requests: memory: "32Gi" cpu: "12" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-sfgg-lkh0k" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-sfgg-lkh0k" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-sfgg-lkh0k in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f b2a59508c11c86ba701c4af25fc536a9475b82c6 # timeout=10 Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_pulsar_test.yaml from git https://github.com/PingCAP-QE/ci.git > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 Avoid second fetch > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 Checking out Revision b2a59508c11c86ba701c4af25fc536a9475b82c6 (origin/main) Avoid second fetch Checking out Revision b2a59508c11c86ba701c4af25fc536a9475b82c6 (origin/main) Commit message: "feat(prow-jobs/pingcap-qe/ci): add periodic job to crawl pull requests and issues (#3449)" Commit message: "feat(prow-jobs/pingcap-qe/ci): add periodic job to crawl pull requests and issues (#3449)" Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_pulsar_test.yaml from git https://github.com/PingCAP-QE/ci.git > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f b2a59508c11c86ba701c4af25fc536a9475b82c6 # timeout=10 Agent pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-9bld-w6hq9 is provisioned from template pingcap_tiflow_pull_cdc_integration_pulsar_test_3484-9bldc-rc0g0 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/3484/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/3484/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "9e38f74d57f6e72da3d999a98852a82d23460f28" jenkins/label: "pingcap_tiflow_pull_cdc_integration_pulsar_test_3484-9bldc" name: "pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-9bld-w6hq9" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/rocky8_golang-1.23:latest" name: "golang" resources: limits: memory: "32Gi" cpu: "12" requests: memory: "32Gi" cpu: "12" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-9bld-w6hq9" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-9bld-w6hq9" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-9bld-w6hq9 in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test > git config core.sparsecheckout # timeout=10 > git checkout -f b2a59508c11c86ba701c4af25fc536a9475b82c6 # timeout=10 Agent pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-qg8g-pl0pp is provisioned from template pingcap_tiflow_pull_cdc_integration_pulsar_test_3484-qg8gt-g1txt --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/3484/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/3484/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "48668d0c298d45724b92bcafa12d187ffb912623" jenkins/label: "pingcap_tiflow_pull_cdc_integration_pulsar_test_3484-qg8gt" name: "pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-qg8g-pl0pp" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/rocky8_golang-1.23:latest" name: "golang" resources: limits: memory: "32Gi" cpu: "12" requests: memory: "32Gi" cpu: "12" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-qg8g-pl0pp" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-qg8g-pl0pp" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-qg8g-pl0pp in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_pulsar_test.yaml from git https://github.com/PingCAP-QE/ci.git Agent pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-jd09-mfgkq is provisioned from template pingcap_tiflow_pull_cdc_integration_pulsar_test_3484-jd09w-b1j7s --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/3484/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/3484/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "9208a39bf47eb99f4fe239d8783665e7ab49c2b6" jenkins/label: "pingcap_tiflow_pull_cdc_integration_pulsar_test_3484-jd09w" name: "pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-jd09-mfgkq" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/rocky8_golang-1.23:latest" name: "golang" resources: limits: memory: "32Gi" cpu: "12" requests: memory: "32Gi" cpu: "12" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-jd09-mfgkq" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-jd09-mfgkq" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-jd09-mfgkq in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_pulsar_test.yaml from git https://github.com/PingCAP-QE/ci.git Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_pulsar_test.yaml from git https://github.com/PingCAP-QE/ci.git Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_pulsar_test.yaml from git https://github.com/PingCAP-QE/ci.git Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-3484/tiflow-cdc) 4030750720 bytes in 8.01 secs (503190156 bytes/sec) [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] sh [Pipeline] { [Pipeline] { [Pipeline] node [Pipeline] node [Pipeline] node [Pipeline] node [Pipeline] node [Pipeline] node [Pipeline] node [Pipeline] node + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh pulsar G00 [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] checkout [Pipeline] checkout Run cases: down_db_require_secure_transport bdr_mode capture_suicide_while_balance_table syncpoint syncpoint_check_ts server_config_compatibility changefeed_dup_error_restart safe_mode kafka_big_messages kafka_compression kafka_messages kafka_sink_error_resume mq_sink_lost_callback mq_sink_dispatcher kafka_column_selector kafka_column_selector_avro debezium lossy_ddl storage_csv_update csv_storage_update_pk_clustered csv_storage_update_pk_nonclustered GIT_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/3484/display/redirect JENKINS_URL=https://do.pingcap.net/jenkins/ EXECUTOR_NUMBER=0 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_pulsar_test","buildid":"1908056220913111045","prowjobid":"b4bfd23f-f137-430c-9c71-ddcc89dd7a6d","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","base_link":"https://github.com/pingcap/tiflow/commit/4c631d5951854ac91765b729fe3849d8e8d5ed8a","pulls":[{"number":12118,"author":"dveeden","sha":"a306f48949e037965a7c1963f500f7e0abbfba51","title":"dm: upgrade go-mysql to v1.12.0","head_ref":"new_gomysql","link":"https://github.com/pingcap/tiflow/pull/12118","commit_link":"https://github.com/pingcap/tiflow/pull/12118/commits/a306f48949e037965a7c1963f500f7e0abbfba51","author_link":"https://github.com/dveeden"}]}} TZ=Asia/Shanghai BUILD_ID=1908056220913111045 POD_LABEL=pingcap_tiflow_pull_cdc_integration_pulsar_test_3484-34gvt HOSTNAME=pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-34gv-66s3n OLDPWD=/home/jenkins RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/3484/display/redirect?page=changes JENKINS_NODE_COOKIE=ccbedc3c-3de8-46ef-ac37-daf94e977e84 GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct JAVA_HOME=/usr/lib/jvm/jre-openjdk JOB_BASE_NAME=pull_cdc_integration_pulsar_test WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test@tmp KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true GIT_URL=https://github.com/PingCAP-QE/ci.git CLASSPATH= KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 FILE_SERVER_URL=http://fileserver.pingcap.net POD_CONTAINER=golang GOPATH=/go BUILD_NUMBER=3484 USE_BAZEL_VERSION=6.5.0 KUBERNETES_PORT=tcp://10.233.0.1:443 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ HOME=/home/jenkins NODE_NAME=pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-34gv-66s3n HUDSON_SERVER_COOKIE=83ef27fe9acccc92 JENKINS_HOME=/var/jenkins_home JOB_NAME=pingcap/tiflow/pull_cdc_integration_pulsar_test RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/3484/display/redirect?page=tests KUBERNETES_SERVICE_PORT_HTTPS=443 KUBERNETES_PORT_443_TCP_PORT=443 GIT_PREVIOUS_COMMIT=3e31ec89ba643faa364a78dbc7f062bb55f9c486 HUDSON_HOME=/var/jenkins_home JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/display/redirect PROW_JOB_ID=b4bfd23f-f137-430c-9c71-ddcc89dd7a6d TEST_GROUP=G00 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/3484/display/redirect?page=artifacts BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/3484/ TERM=xterm STAGE_NAME=Test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/ BUILD_DISPLAY_NAME=#3484 SHLVL=5 GIT_BRANCH=origin/main BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-3484 KUBERNETES_SERVICE_PORT=443 NODE_LABELS=pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-34gv-66s3n pingcap_tiflow_pull_cdc_integration_pulsar_test_3484-34gvt PATH=/go/bin:/root/.cargo/bin:/usr/local/go/bin/:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/gradle-7.4.2/bin:/opt/apache-maven-3.8.8/bin:/usr/local/pulsar/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../../scripts/bin GIT_PREVIOUS_SUCCESSFUL_COMMIT=3e31ec89ba643faa364a78dbc7f062bb55f9c486 KUBERNETES_SERVICE_HOST=10.233.0.1 JENKINS_SERVER_COOKIE=durable-1c247575f7a5458f3a8df09baf43395940069f8be33f18a8fa953544ffab3376 _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/down_db_require_secure_transport/run.sh using Sink-Type: pulsar... <<================= find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/bdr_mode/run.sh using Sink-Type: pulsar... <<================= [Fri Apr 4 15:30:47 CST 2025] <<<<<< run test case bdr_mode success! >>>>>> The recommended git tool is: git [Pipeline] container [Pipeline] { The recommended git tool is: git [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] { [Pipeline] checkout [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git The recommended git tool is: git [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-b84s-w4c6p Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-6nlr-p88d4 [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-qj98-cgzmv [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-vgqp-h2d14 [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-2xgw-gnw10 Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-j3lw-c6tm2 [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-7fxb-ntnhx [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow [Pipeline] { Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-5mln-gcmj2 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@356f4fed; decorates RemoteLauncher[hudson.remoting.Channel@3b2638c9:JNLP4-connect connection from 10.233.86.20/10.233.86.20:44596] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 [Pipeline] cache No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@72a5fd8e; decorates RemoteLauncher[hudson.remoting.Channel@58bd894f:JNLP4-connect connection from 10.233.127.211/10.233.127.211:33434] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@7e66f3b8; decorates RemoteLauncher[hudson.remoting.Channel@14a9c592:JNLP4-connect connection from 10.233.97.117/10.233.97.117:47546] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@6e22a0b2; decorates RemoteLauncher[hudson.remoting.Channel@62e66e5e:JNLP4-connect connection from 10.233.71.242/10.233.71.242:60136] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@3b5d2473; decorates RemoteLauncher[hudson.remoting.Channel@25b89481:JNLP4-connect connection from 10.233.84.40/10.233.84.40:32924] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@44fccf8d; decorates RemoteLauncher[hudson.remoting.Channel@351cdd10:JNLP4-connect connection from 10.233.69.189/10.233.69.189:38370] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/capture_suicide_while_balance_table/run.sh using Sink-Type: pulsar... <<================= [Fri Apr 4 15:30:50 CST 2025] <<<<<< run test case capture_suicide_while_balance_table success! >>>>>> > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 Avoid second fetch > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 Checking out Revision b2a59508c11c86ba701c4af25fc536a9475b82c6 (origin/main) Avoid second fetch Checking out Revision b2a59508c11c86ba701c4af25fc536a9475b82c6 (origin/main) Avoid second fetch Checking out Revision b2a59508c11c86ba701c4af25fc536a9475b82c6 (origin/main) Agent pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-qj98-cgzmv is provisioned from template pingcap_tiflow_pull_cdc_integration_pulsar_test_3484-qj98c-msw9w --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/3484/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/3484/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "92eb81382ea1526d90c6c00cc8310b53dd477e93" jenkins/label: "pingcap_tiflow_pull_cdc_integration_pulsar_test_3484-qj98c" name: "pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-qj98-cgzmv" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/rocky8_golang-1.23:latest" name: "golang" resources: limits: memory: "32Gi" cpu: "12" requests: memory: "32Gi" cpu: "12" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-qj98-cgzmv" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-qj98-cgzmv" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Agent pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-b84s-w4c6p is provisioned from template pingcap_tiflow_pull_cdc_integration_pulsar_test_3484-b84s6-11pt2 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/3484/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/3484/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "6c3241e2f14786e14a7eba954907c039162cd461" jenkins/label: "pingcap_tiflow_pull_cdc_integration_pulsar_test_3484-b84s6" name: "pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-b84s-w4c6p" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/rocky8_golang-1.23:latest" name: "golang" resources: limits: memory: "32Gi" cpu: "12" requests: memory: "32Gi" cpu: "12" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-b84s-w4c6p" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-b84s-w4c6p" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-qj98-cgzmv in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test Running on pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-b84s-w4c6p in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test Commit message: "feat(prow-jobs/pingcap-qe/ci): add periodic job to crawl pull requests and issues (#3449)" Commit message: "feat(prow-jobs/pingcap-qe/ci): add periodic job to crawl pull requests and issues (#3449)" Commit message: "feat(prow-jobs/pingcap-qe/ci): add periodic job to crawl pull requests and issues (#3449)" > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 Avoid second fetch > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f b2a59508c11c86ba701c4af25fc536a9475b82c6 # timeout=10 Checking out Revision b2a59508c11c86ba701c4af25fc536a9475b82c6 (origin/main) Avoid second fetch Checking out Revision b2a59508c11c86ba701c4af25fc536a9475b82c6 (origin/main) Avoid second fetch Checking out Revision b2a59508c11c86ba701c4af25fc536a9475b82c6 (origin/main) Commit message: "feat(prow-jobs/pingcap-qe/ci): add periodic job to crawl pull requests and issues (#3449)" Commit message: "feat(prow-jobs/pingcap-qe/ci): add periodic job to crawl pull requests and issues (#3449)" Commit message: "feat(prow-jobs/pingcap-qe/ci): add periodic job to crawl pull requests and issues (#3449)" Agent pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-7fxb-ntnhx is provisioned from template pingcap_tiflow_pull_cdc_integration_pulsar_test_3484-7fxbb-sbb9f --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/3484/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/3484/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "574cccd8be07bba05a548311da67cfe5ffce89f6" jenkins/label: "pingcap_tiflow_pull_cdc_integration_pulsar_test_3484-7fxbb" name: "pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-7fxb-ntnhx" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/rocky8_golang-1.23:latest" name: "golang" resources: limits: memory: "32Gi" cpu: "12" requests: memory: "32Gi" cpu: "12" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-7fxb-ntnhx" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-7fxb-ntnhx" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-7fxb-ntnhx in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test Agent pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-vgqp-h2d14 is provisioned from template pingcap_tiflow_pull_cdc_integration_pulsar_test_3484-vgqpb-vk2qf --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/3484/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/3484/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "9e18ba05eeae698e50f13465aabbb28fbe19af7a" jenkins/label: "pingcap_tiflow_pull_cdc_integration_pulsar_test_3484-vgqpb" name: "pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-vgqp-h2d14" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/rocky8_golang-1.23:latest" name: "golang" resources: limits: memory: "32Gi" cpu: "12" requests: memory: "32Gi" cpu: "12" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-vgqp-h2d14" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-vgqp-h2d14" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-vgqp-h2d14 in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test Agent pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-j3lw-c6tm2 is provisioned from template pingcap_tiflow_pull_cdc_integration_pulsar_test_3484-j3lwh-hghj4 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/3484/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/3484/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "40bb5b7acdcd61159f869d2e1382736a42df3160" jenkins/label: "pingcap_tiflow_pull_cdc_integration_pulsar_test_3484-j3lwh" name: "pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-j3lw-c6tm2" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/rocky8_golang-1.23:latest" name: "golang" resources: limits: memory: "32Gi" cpu: "12" requests: memory: "32Gi" cpu: "12" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-j3lw-c6tm2" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-j3lw-c6tm2" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-j3lw-c6tm2 in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test Agent pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-5mln-gcmj2 is provisioned from template pingcap_tiflow_pull_cdc_integration_pulsar_test_3484-5mlns-lrwg9 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/3484/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/3484/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "92943543abf635e9afe84bfc176778eb3a0537c8" jenkins/label: "pingcap_tiflow_pull_cdc_integration_pulsar_test_3484-5mlns" name: "pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-5mln-gcmj2" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/rocky8_golang-1.23:latest" name: "golang" resources: limits: memory: "32Gi" cpu: "12" requests: memory: "32Gi" cpu: "12" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-5mln-gcmj2" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-5mln-gcmj2" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f b2a59508c11c86ba701c4af25fc536a9475b82c6 # timeout=10 Running on pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-5mln-gcmj2 in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test Agent pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-6nlr-p88d4 is provisioned from template pingcap_tiflow_pull_cdc_integration_pulsar_test_3484-6nlr7-pdb6v --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/3484/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/3484/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "7e09925edcc75906310a11706a50690da257ba20" jenkins/label: "pingcap_tiflow_pull_cdc_integration_pulsar_test_3484-6nlr7" name: "pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-6nlr-p88d4" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/rocky8_golang-1.23:latest" name: "golang" resources: limits: memory: "32Gi" cpu: "12" requests: memory: "32Gi" cpu: "12" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-6nlr-p88d4" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-6nlr-p88d4" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" > git config core.sparsecheckout # timeout=10 > git checkout -f b2a59508c11c86ba701c4af25fc536a9475b82c6 # timeout=10 Running on pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-6nlr-p88d4 in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f b2a59508c11c86ba701c4af25fc536a9475b82c6 # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f b2a59508c11c86ba701c4af25fc536a9475b82c6 # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f b2a59508c11c86ba701c4af25fc536a9475b82c6 # timeout=10 Agent pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-2xgw-gnw10 is provisioned from template pingcap_tiflow_pull_cdc_integration_pulsar_test_3484-2xgwq-kg9mq --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/3484/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/3484/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "0214d6b08e7a18df70d6591ee23552a260b37c76" jenkins/label: "pingcap_tiflow_pull_cdc_integration_pulsar_test_3484-2xgwq" name: "pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-2xgw-gnw10" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/rocky8_golang-1.23:latest" name: "golang" resources: limits: memory: "32Gi" cpu: "12" requests: memory: "32Gi" cpu: "12" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-2xgw-gnw10" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-2xgw-gnw10" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-2xgw-gnw10 in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/syncpoint/run.sh using Sink-Type: pulsar... <<================= kafka downstream isn't support syncpoint record [Fri Apr 4 15:30:54 CST 2025] <<<<<< run test case syncpoint success! >>>>>> find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/syncpoint_check_ts/run.sh using Sink-Type: pulsar... <<================= kafka downstream isn't support syncpoint record [Fri Apr 4 15:30:57 CST 2025] <<<<<< run test case syncpoint_check_ts success! >>>>>> Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-3484/tiflow-cdc) 4030750720 bytes in 8.15 secs (494331240 bytes/sec) [Pipeline] { [Pipeline] cache find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/server_config_compatibility/run.sh using Sink-Type: pulsar... <<================= [Fri Apr 4 15:31:00 CST 2025] <<<<<< run test case server_config_compatibility success! >>>>>> find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/changefeed_dup_error_restart/run.sh using Sink-Type: pulsar... <<================= [Fri Apr 4 15:31:03 CST 2025] <<<<<< run test case changefeed_dup_error_restart success! >>>>>> find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/safe_mode/run.sh using Sink-Type: pulsar... <<================= [Fri Apr 4 15:31:06 CST 2025] <<<<<< run test case safe_mode success! >>>>>> Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-3484/tiflow-cdc) 4030750720 bytes in 8.59 secs (469346867 bytes/sec) [Pipeline] { [Pipeline] cache find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/kafka_big_messages/run.sh using Sink-Type: pulsar... <<================= [Fri Apr 4 15:31:09 CST 2025] <<<<<< run test case kafka_big_messages success! >>>>>> find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/kafka_compression/run.sh using Sink-Type: pulsar... <<================= [Fri Apr 4 15:31:13 CST 2025] <<<<<< run test case kafka_compression success! >>>>>> find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/kafka_messages/run.sh using Sink-Type: pulsar... <<================= [Fri Apr 4 15:31:16 CST 2025] <<<<<< run test case kafka_messages success! >>>>>> Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-3484/tiflow-cdc) 4030750720 bytes in 7.36 secs (547850263 bytes/sec) [Pipeline] { [Pipeline] sh [Pipeline] sh find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/kafka_sink_error_resume/run.sh using Sink-Type: pulsar... <<================= [Fri Apr 4 15:31:19 CST 2025] <<<<<< run test case kafka_sink_error_resume success! >>>>>> [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh pulsar G01 Run cases: http_api http_api_tls api_v2 http_api_tls_with_user_auth cli_tls_with_auth debezium_basic kafka_simple_basic kafka_simple_basic_avro kafka_simple_handle_key_only kafka_simple_handle_key_only_avro kafka_simple_claim_check kafka_simple_claim_check_avro canal_json_adapter_compatibility canal_json_basic canal_json_content_compatible multi_topics avro_basic canal_json_handle_key_only open_protocol_handle_key_only canal_json_claim_check open_protocol_claim_check canal_json_storage_basic canal_json_storage_partition_table multi_tables_ddl GIT_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/3484/display/redirect JENKINS_URL=https://do.pingcap.net/jenkins/ EXECUTOR_NUMBER=0 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_pulsar_test","buildid":"1908056220913111045","prowjobid":"b4bfd23f-f137-430c-9c71-ddcc89dd7a6d","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","base_link":"https://github.com/pingcap/tiflow/commit/4c631d5951854ac91765b729fe3849d8e8d5ed8a","pulls":[{"number":12118,"author":"dveeden","sha":"a306f48949e037965a7c1963f500f7e0abbfba51","title":"dm: upgrade go-mysql to v1.12.0","head_ref":"new_gomysql","link":"https://github.com/pingcap/tiflow/pull/12118","commit_link":"https://github.com/pingcap/tiflow/pull/12118/commits/a306f48949e037965a7c1963f500f7e0abbfba51","author_link":"https://github.com/dveeden"}]}} TZ=Asia/Shanghai BUILD_ID=1908056220913111045 POD_LABEL=pingcap_tiflow_pull_cdc_integration_pulsar_test_3484-0lxqm HOSTNAME=pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-0lxq-6vnt9 OLDPWD=/home/jenkins RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/3484/display/redirect?page=changes JENKINS_NODE_COOKIE=1e7ea4fe-50e7-44e4-8d74-2e78e0dc5182 GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct JAVA_HOME=/usr/lib/jvm/jre-openjdk JOB_BASE_NAME=pull_cdc_integration_pulsar_test WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test@tmp KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true GIT_URL=https://github.com/PingCAP-QE/ci.git CLASSPATH= KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 FILE_SERVER_URL=http://fileserver.pingcap.net POD_CONTAINER=golang GOPATH=/go BUILD_NUMBER=3484 USE_BAZEL_VERSION=6.5.0 KUBERNETES_PORT=tcp://10.233.0.1:443 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ HOME=/home/jenkins NODE_NAME=pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-0lxq-6vnt9 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 JENKINS_HOME=/var/jenkins_home JOB_NAME=pingcap/tiflow/pull_cdc_integration_pulsar_test RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/3484/display/redirect?page=tests KUBERNETES_SERVICE_PORT_HTTPS=443 KUBERNETES_PORT_443_TCP_PORT=443 GIT_PREVIOUS_COMMIT=3e31ec89ba643faa364a78dbc7f062bb55f9c486 HUDSON_HOME=/var/jenkins_home JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/display/redirect PROW_JOB_ID=b4bfd23f-f137-430c-9c71-ddcc89dd7a6d TEST_GROUP=G01 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/3484/display/redirect?page=artifacts BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/3484/ TERM=xterm STAGE_NAME=Test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/ BUILD_DISPLAY_NAME=#3484 SHLVL=5 GIT_BRANCH=origin/main BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-3484 KUBERNETES_SERVICE_PORT=443 NODE_LABELS=pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-0lxq-6vnt9 pingcap_tiflow_pull_cdc_integration_pulsar_test_3484-0lxqm PATH=/go/bin:/root/.cargo/bin:/usr/local/go/bin/:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/gradle-7.4.2/bin:/opt/apache-maven-3.8.8/bin:/usr/local/pulsar/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../../scripts/bin GIT_PREVIOUS_SUCCESSFUL_COMMIT=3e31ec89ba643faa364a78dbc7f062bb55f9c486 KUBERNETES_SERVICE_HOST=10.233.0.1 JENKINS_SERVER_COOKIE=durable-1c247575f7a5458f3a8df09baf43395940069f8be33f18a8fa953544ffab3376 _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/http_api/run.sh using Sink-Type: pulsar... <<================= [Fri Apr 4 15:31:20 CST 2025] <<<<<< run test case http_api success! >>>>>> + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh pulsar G03 Run cases: row_format drop_many_tables processor_stop_delay partition_table ddl_with_exists GIT_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/3484/display/redirect JENKINS_URL=https://do.pingcap.net/jenkins/ EXECUTOR_NUMBER=0 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_pulsar_test","buildid":"1908056220913111045","prowjobid":"b4bfd23f-f137-430c-9c71-ddcc89dd7a6d","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","base_link":"https://github.com/pingcap/tiflow/commit/4c631d5951854ac91765b729fe3849d8e8d5ed8a","pulls":[{"number":12118,"author":"dveeden","sha":"a306f48949e037965a7c1963f500f7e0abbfba51","title":"dm: upgrade go-mysql to v1.12.0","head_ref":"new_gomysql","link":"https://github.com/pingcap/tiflow/pull/12118","commit_link":"https://github.com/pingcap/tiflow/pull/12118/commits/a306f48949e037965a7c1963f500f7e0abbfba51","author_link":"https://github.com/dveeden"}]}} TZ=Asia/Shanghai BUILD_ID=1908056220913111045 POD_LABEL=pingcap_tiflow_pull_cdc_integration_pulsar_test_3484-gr29d HOSTNAME=pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-gr29-fxw7k OLDPWD=/home/jenkins RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/3484/display/redirect?page=changes JENKINS_NODE_COOKIE=c9daa04d-2766-44e2-a78b-3ac3c1e6c893 GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct JAVA_HOME=/usr/lib/jvm/jre-openjdk JOB_BASE_NAME=pull_cdc_integration_pulsar_test WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test@tmp KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true GIT_URL=https://github.com/PingCAP-QE/ci.git CLASSPATH= KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 FILE_SERVER_URL=http://fileserver.pingcap.net POD_CONTAINER=golang GOPATH=/go BUILD_NUMBER=3484 USE_BAZEL_VERSION=6.5.0 KUBERNETES_PORT=tcp://10.233.0.1:443 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ HOME=/home/jenkins NODE_NAME=pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-gr29-fxw7k HUDSON_SERVER_COOKIE=83ef27fe9acccc92 JENKINS_HOME=/var/jenkins_home JOB_NAME=pingcap/tiflow/pull_cdc_integration_pulsar_test RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/3484/display/redirect?page=tests KUBERNETES_SERVICE_PORT_HTTPS=443 KUBERNETES_PORT_443_TCP_PORT=443 GIT_PREVIOUS_COMMIT=3e31ec89ba643faa364a78dbc7f062bb55f9c486 HUDSON_HOME=/var/jenkins_home JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/display/redirect PROW_JOB_ID=b4bfd23f-f137-430c-9c71-ddcc89dd7a6d TEST_GROUP=G03 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/3484/display/redirect?page=artifacts BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/3484/ TERM=xterm STAGE_NAME=Test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/ BUILD_DISPLAY_NAME=#3484 SHLVL=5 GIT_BRANCH=origin/main BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-3484 KUBERNETES_SERVICE_PORT=443 NODE_LABELS=pingcap_tiflow_pull_cdc_integration_pulsar_test_3484-gr29d pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-gr29-fxw7k PATH=/go/bin:/root/.cargo/bin:/usr/local/go/bin/:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/gradle-7.4.2/bin:/opt/apache-maven-3.8.8/bin:/usr/local/pulsar/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../../scripts/bin GIT_PREVIOUS_SUCCESSFUL_COMMIT=3e31ec89ba643faa364a78dbc7f062bb55f9c486 KUBERNETES_SERVICE_HOST=10.233.0.1 + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test JENKINS_SERVER_COOKIE=durable-1c247575f7a5458f3a8df09baf43395940069f8be33f18a8fa953544ffab3376 _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh pulsar G02 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/row_format/run.sh using Sink-Type: pulsar... <<================= The 1 times to try to start tidb cluster... [Pipeline] { [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { Run cases: consistent_replicate_ddl consistent_replicate_gbk consistent_replicate_nfs consistent_replicate_storage_file consistent_replicate_storage_file_large_value consistent_replicate_storage_s3 consistent_partition_table kafka_big_messages_v2 multi_tables_ddl_v2 multi_topics_v2 storage_cleanup csv_storage_basic csv_storage_multi_tables_ddl csv_storage_partition_table GIT_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/3484/display/redirect JENKINS_URL=https://do.pingcap.net/jenkins/ EXECUTOR_NUMBER=0 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_pulsar_test","buildid":"1908056220913111045","prowjobid":"b4bfd23f-f137-430c-9c71-ddcc89dd7a6d","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","base_link":"https://github.com/pingcap/tiflow/commit/4c631d5951854ac91765b729fe3849d8e8d5ed8a","pulls":[{"number":12118,"author":"dveeden","sha":"a306f48949e037965a7c1963f500f7e0abbfba51","title":"dm: upgrade go-mysql to v1.12.0","head_ref":"new_gomysql","link":"https://github.com/pingcap/tiflow/pull/12118","commit_link":"https://github.com/pingcap/tiflow/pull/12118/commits/a306f48949e037965a7c1963f500f7e0abbfba51","author_link":"https://github.com/dveeden"}]}} TZ=Asia/Shanghai BUILD_ID=1908056220913111045 POD_LABEL=pingcap_tiflow_pull_cdc_integration_pulsar_test_3484-km3gp HOSTNAME=pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-km3g-0ld68 OLDPWD=/home/jenkins RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/3484/display/redirect?page=changes JENKINS_NODE_COOKIE=5e7c2896-20f2-4382-9d2a-855d664e4109 GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct JAVA_HOME=/usr/lib/jvm/jre-openjdk JOB_BASE_NAME=pull_cdc_integration_pulsar_test WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test@tmp KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true GIT_URL=https://github.com/PingCAP-QE/ci.git CLASSPATH= KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 FILE_SERVER_URL=http://fileserver.pingcap.net POD_CONTAINER=golang GOPATH=/go BUILD_NUMBER=3484 USE_BAZEL_VERSION=6.5.0 KUBERNETES_PORT=tcp://10.233.0.1:443 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ HOME=/home/jenkins NODE_NAME=pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-km3g-0ld68 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 JENKINS_HOME=/var/jenkins_home JOB_NAME=pingcap/tiflow/pull_cdc_integration_pulsar_test RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/3484/display/redirect?page=tests KUBERNETES_SERVICE_PORT_HTTPS=443 KUBERNETES_PORT_443_TCP_PORT=443 GIT_PREVIOUS_COMMIT=3e31ec89ba643faa364a78dbc7f062bb55f9c486 HUDSON_HOME=/var/jenkins_home JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/display/redirect PROW_JOB_ID=b4bfd23f-f137-430c-9c71-ddcc89dd7a6d TEST_GROUP=G02 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/3484/display/redirect?page=artifacts BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/3484/ TERM=xterm STAGE_NAME=Test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/ BUILD_DISPLAY_NAME=#3484 SHLVL=5 GIT_BRANCH=origin/main BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-3484 KUBERNETES_SERVICE_PORT=443 NODE_LABELS=pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-km3g-0ld68 pingcap_tiflow_pull_cdc_integration_pulsar_test_3484-km3gp PATH=/go/bin:/root/.cargo/bin:/usr/local/go/bin/:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/gradle-7.4.2/bin:/opt/apache-maven-3.8.8/bin:/usr/local/pulsar/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../../scripts/bin GIT_PREVIOUS_SUCCESSFUL_COMMIT=3e31ec89ba643faa364a78dbc7f062bb55f9c486 KUBERNETES_SERVICE_HOST=10.233.0.1 JENKINS_SERVER_COOKIE=durable-1c247575f7a5458f3a8df09baf43395940069f8be33f18a8fa953544ffab3376 _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/consistent_replicate_ddl/run.sh using Sink-Type: pulsar... <<================= [Fri Apr 4 15:31:20 CST 2025] <<<<<< run test case consistent_replicate_ddl success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] checkout [Pipeline] checkout The recommended git tool is: git The recommended git tool is: git [Pipeline] checkout [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git The recommended git tool is: git [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow [Pipeline] { No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@15d29f46; decorates RemoteLauncher[hudson.remoting.Channel@4973ac5b:JNLP4-connect connection from 10.233.77.248/10.233.77.248:51252] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 [Pipeline] cache No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@1fb11232; decorates RemoteLauncher[hudson.remoting.Channel@6d01608c:JNLP4-connect connection from 10.233.67.246/10.233.67.246:42806] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@1c5e047e; decorates RemoteLauncher[hudson.remoting.Channel@211cb240:JNLP4-connect connection from 10.233.108.225/10.233.108.225:48356] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@6f42b835; decorates RemoteLauncher[hudson.remoting.Channel@4f0bf105:JNLP4-connect connection from 10.233.75.66/10.233.75.66:60972] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@336adf01; decorates RemoteLauncher[hudson.remoting.Channel@27cb1611:JNLP4-connect connection from 10.233.74.193/10.233.74.193:33592] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@16b64850; decorates RemoteLauncher[hudson.remoting.Channel@6397cf5:JNLP4-connect connection from 10.233.78.177/10.233.78.177:35046] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@142c6931; decorates RemoteLauncher[hudson.remoting.Channel@7418db8:JNLP4-connect connection from 10.233.93.19/10.233.93.19:37190] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@4ce49662; decorates RemoteLauncher[hudson.remoting.Channel@148c826f:JNLP4-connect connection from 10.233.66.184/10.233.66.184:39524] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/mq_sink_lost_callback/run.sh using Sink-Type: pulsar... <<================= [Fri Apr 4 15:31:22 CST 2025] <<<<<< run test case mq_sink_lost_callback success! >>>>>> start tidb cluster in /tmp/tidb_cdc_test/row_format Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... Avoid second fetch Avoid second fetch Checking out Revision b2a59508c11c86ba701c4af25fc536a9475b82c6 (origin/main) Checking out Revision b2a59508c11c86ba701c4af25fc536a9475b82c6 (origin/main) Avoid second fetch Checking out Revision b2a59508c11c86ba701c4af25fc536a9475b82c6 (origin/main) Avoid second fetch Checking out Revision b2a59508c11c86ba701c4af25fc536a9475b82c6 (origin/main) > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f b2a59508c11c86ba701c4af25fc536a9475b82c6 # timeout=10 find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/http_api_tls/run.sh using Sink-Type: pulsar... <<================= [Fri Apr 4 15:31:23 CST 2025] <<<<<< run test case http_api_tls success! >>>>>> > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f b2a59508c11c86ba701c4af25fc536a9475b82c6 # timeout=10 Commit message: "feat(prow-jobs/pingcap-qe/ci): add periodic job to crawl pull requests and issues (#3449)" Avoid second fetch Checking out Revision b2a59508c11c86ba701c4af25fc536a9475b82c6 (origin/main) Commit message: "feat(prow-jobs/pingcap-qe/ci): add periodic job to crawl pull requests and issues (#3449)" Commit message: "feat(prow-jobs/pingcap-qe/ci): add periodic job to crawl pull requests and issues (#3449)" Commit message: "feat(prow-jobs/pingcap-qe/ci): add periodic job to crawl pull requests and issues (#3449)" > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f b2a59508c11c86ba701c4af25fc536a9475b82c6 # timeout=10 Avoid second fetch Checking out Revision b2a59508c11c86ba701c4af25fc536a9475b82c6 (origin/main) > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f b2a59508c11c86ba701c4af25fc536a9475b82c6 # timeout=10 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f b2a59508c11c86ba701c4af25fc536a9475b82c6 # timeout=10 Avoid second fetch Checking out Revision b2a59508c11c86ba701c4af25fc536a9475b82c6 (origin/main) > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f b2a59508c11c86ba701c4af25fc536a9475b82c6 # timeout=10 Commit message: "feat(prow-jobs/pingcap-qe/ci): add periodic job to crawl pull requests and issues (#3449)" > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f b2a59508c11c86ba701c4af25fc536a9475b82c6 # timeout=10 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 Commit message: "feat(prow-jobs/pingcap-qe/ci): add periodic job to crawl pull requests and issues (#3449)" Avoid second fetch Checking out Revision b2a59508c11c86ba701c4af25fc536a9475b82c6 (origin/main) Commit message: "feat(prow-jobs/pingcap-qe/ci): add periodic job to crawl pull requests and issues (#3449)" find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/consistent_replicate_gbk/run.sh using Sink-Type: pulsar... <<================= * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 24927 failed: Connection refused * Failed to connect to 127.0.0.1 port 24927: Connection refused * Closing connection 0 You are running an older version of MinIO released 4 years ago Update: Run `mc admin update` Attempting encryption of all config, IAM users and policies on MinIO backend Commit message: "feat(prow-jobs/pingcap-qe/ci): add periodic job to crawl pull requests and issues (#3449)" Endpoint: http://127.0.0.1:24927 Object API (Amazon S3 compatible): Go: https://docs.min.io/docs/golang-client-quickstart-guide Java: https://docs.min.io/docs/java-client-quickstart-guide Python: https://docs.min.io/docs/python-client-quickstart-guide JavaScript: https://docs.min.io/docs/javascript-client-quickstart-guide .NET: https://docs.min.io/docs/dotnet-client-quickstart-guide > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f b2a59508c11c86ba701c4af25fc536a9475b82c6 # timeout=10 * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 24927 (#0) > GET / HTTP/1.1 > Host: 127.0.0.1:24927 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 403 Forbidden < Accept-Ranges: bytes < Content-Length: 226 < Content-Security-Policy: block-all-mixed-content < Content-Type: application/xml < Server: MinIO/RELEASE.2020-07-27T18-37-02Z < Vary: Origin < X-Amz-Request-Id: 18330D19FB42A964 < X-Xss-Protection: 1; mode=block < Date: Fri, 04 Apr 2025 07:31:25 GMT < { [226 bytes data] * Connection #0 to host 127.0.0.1 left intact find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/mq_sink_dispatcher/run.sh using Sink-Type: pulsar... <<================= [Fri Apr 4 15:31:26 CST 2025] <<<<<< run test case mq_sink_dispatcher success! >>>>>> Bucket 's3://logbucket/' created [Fri Apr 4 15:31:26 CST 2025] <<<<<< run test case consistent_replicate_gbk success! >>>>>> Exiting on signal: INTERRUPT find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/api_v2/run.sh using Sink-Type: pulsar... <<================= Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/kafka_column_selector/run.sh using Sink-Type: pulsar... <<================= [Fri Apr 4 15:31:29 CST 2025] <<<<<< run test case kafka_column_selector success! >>>>>> =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/consistent_replicate_nfs/run.sh using Sink-Type: pulsar... <<================= [Fri Apr 4 15:31:29 CST 2025] <<<<<< run test case consistent_replicate_nfs success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-3484/tiflow-cdc) 4030750720 bytes in 6.69 secs (602263419 bytes/sec) [Pipeline] { [Pipeline] cache find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/http_api_tls_with_user_auth/run.sh using Sink-Type: pulsar... <<================= [Fri Apr 4 15:31:30 CST 2025] <<<<<< run test case http_api_tls_with_user_auth success! >>>>>> VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489357325317402269 TiDB Cluster ID. tikv_gc_leader_uuid 657fed92304000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-gr29-fxw7k, pid:1487, start at 2025-04-04 15:31:31.16539765 +0800 CST m=+1.465227315 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:33:31.173 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:31:31.137 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:21:31.137 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489357325317402269 TiDB Cluster ID. tikv_gc_leader_uuid 657fed92304000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-gr29-fxw7k, pid:1487, start at 2025-04-04 15:31:31.16539765 +0800 CST m=+1.465227315 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:33:31.173 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:31:31.137 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:21:31.137 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489357325914821558 TiDB Cluster ID. tikv_gc_leader_uuid 657fed923dc0005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-gr29-fxw7k, pid:1564, start at 2025-04-04 15:31:31.197704852 +0800 CST m=+1.424571070 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:33:31.205 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:31:31.191 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:21:31.191 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/row_format/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/row_format/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/row_format/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/row_format/tiflash/log/proxy.log"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/row_format/tiflash-proxy.toml"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/kafka_column_selector_avro/run.sh using Sink-Type: pulsar... <<================= [Fri Apr 4 15:31:32 CST 2025] <<<<<< run test case kafka_column_selector_avro success! >>>>>> find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/consistent_replicate_storage_file/run.sh using Sink-Type: pulsar... <<================= [Fri Apr 4 15:31:32 CST 2025] <<<<<< run test case consistent_replicate_storage_file success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/cli_tls_with_auth/run.sh using Sink-Type: pulsar... <<================= + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.row_format.cli.2861.out cli tso query --pd=http://127.0.0.1:2379 find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/debezium/run.sh using Sink-Type: pulsar... <<================= find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/lossy_ddl/run.sh using Sink-Type: pulsar... <<================= [Fri Apr 4 15:31:35 CST 2025] <<<<<< run test case lossy_ddl success! >>>>>> + set +x + tso='457114096602447873 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 457114096602447873 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 4 15:31:35 CST 2025] <<<<<< START cdc server in row_format case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.row_format.29022904.out server --log-file /tmp/tidb_cdc_test/row_format/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/row_format/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/consistent_replicate_storage_file_large_value/run.sh using Sink-Type: pulsar... <<================= [Fri Apr 4 15:31:36 CST 2025] <<<<<< run test case consistent_replicate_storage_file_large_value success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/debezium_basic/run.sh using Sink-Type: pulsar... <<================= [Fri Apr 4 15:31:37 CST 2025] <<<<<< run test case debezium_basic success! >>>>>> find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/storage_csv_update/run.sh using Sink-Type: pulsar... <<================= [Fri Apr 4 15:31:38 CST 2025] <<<<<< run test case storage_csv_update success! >>>>>> + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 07:31:38 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [817 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6cb6f65d-8564-4e51-a871-671c79181b32 {"id":"6cb6f65d-8564-4e51-a871-671c79181b32","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743751896} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffb62b3dd5 6cb6f65d-8564-4e51-a871-671c79181b32 /tidb/cdc/default/default/upstream/7489357325317402269 {"id":7489357325317402269,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6cb6f65d-8564-4e51-a871-671c79181b32 {"id":"6cb6f65d-8564-4e51-a871-671c79181b32","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743751896} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffb62b3dd5 6cb6f65d-8564-4e51-a871-671c79181b32 /tidb/cdc/default/default/upstream/7489357325317402269 {"id":7489357325317402269,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6cb6f65d-8564-4e51-a871-671c79181b32 {"id":"6cb6f65d-8564-4e51-a871-671c79181b32","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743751896} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffb62b3dd5 6cb6f65d-8564-4e51-a871-671c79181b32 /tidb/cdc/default/default/upstream/7489357325317402269 {"id":7489357325317402269,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x ++ date + echo '[Fri Apr 4 15:31:38 CST 2025] <<<<<< START pulsar cluster in row_format case >>>>>>' [Fri Apr 4 15:31:38 CST 2025] <<<<<< START pulsar cluster in row_format case >>>>>> + workdir=/tmp/tidb_cdc_test/row_format + cluster_type=normal + cd /tmp/tidb_cdc_test/row_format + DEFAULT_PULSAR_HOME=/usr/local/pulsar + pulsar_dir=/usr/local/pulsar ++ cat + mtls_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderTls brokerClientTlsEnabled=true brokerClientTrustCertsFilePath=/tmp/tidb_cdc_test/row_format/ca.cert.pem brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls brokerClientAuthenticationParameters={"tlsCertFile":"/tmp/tidb_cdc_test/row_format/broker_client.cert.pem","tlsKeyFile":"/tmp/tidb_cdc_test/row_format/broker_client.key-pk8.pem"} brokerServicePortTls=6651 webServicePortTls=8443 tlsTrustCertsFilePath=/tmp/tidb_cdc_test/row_format/ca.cert.pem tlsCertificateFilePath=/tmp/tidb_cdc_test/row_format/server.cert.pem tlsKeyFilePath=/tmp/tidb_cdc_test/row_format/server.key-pk8.pem tlsRequireTrustedClientCertOnConnect=true tlsAllowInsecureConnection=false tlsCertRefreshCheckDurationSec=300' ++ cat + normal_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' ++ cat + mtls_client_conf=' webServiceUrl=https://localhost:8443/ brokerServiceUrl=pulsar+ssl://localhost:6651/ authPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls authParams=tlsCertFile:/tmp/tidb_cdc_test/row_format/broker_client.cert.pem,tlsKeyFile:/tmp/tidb_cdc_test/row_format/broker_client.key-pk8.pem tlsTrustCertsFilePath=/tmp/tidb_cdc_test/row_format/ca.cert.pem' ++ cat + oauth_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/ authPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 authParams={"privateKey":"/tmp/tidb_cdc_test/row_format/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"}' ++ cat + oauth_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderToken brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 brokerClientAuthenticationParameters={"privateKey":"file:///tmp/tidb_cdc_test/row_format/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"} tokenSecretKey=data:;base64,U0poWDM2X0thcFlTeWJCdEpxMzVseFhfQnJyNExSVVJTa203UW1YSkdteThwVUZXOUVJT2NWUVBzeWt6OS1qag==' ++ cat + credential_json=' { "client_id":"1234", "client_secret":"e0KVlA2EiBfjoN13olyZd2kv1KL", "audience":"cdc-api-uri", "issuer_url":"http://localhost:9096", "type": "client_credentials" }' ++ cat + cert_server_conf='[ req ] default_bits = 2048 prompt = no default_md = sha256 distinguished_name = dn [ v3_ext ] authorityKeyIdentifier=keyid,issuer:always basicConstraints=CA:FALSE keyUsage=critical, digitalSignature, keyEncipherment extendedKeyUsage=serverAuth subjectAltName=@alt_names [ dn ] CN = server [ alt_names ] DNS.1 = localhost IP.1 = 127.0.0.1' + echo ' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' + cp /usr/local/pulsar/conf/standalone.conf /tmp/tidb_cdc_test/row_format/pulsar_standalone.conf + pulsar_port=6650 + '[' normal == mtls ']' + '[' normal == oauth ']' + echo 'no cluster type specified, using default configuration.' no cluster type specified, using default configuration. ++ date + echo '[Fri Apr 4 15:31:38 CST 2025] <<<<<< START pulsar cluster in normal mode in row_format case >>>>>>' [Fri Apr 4 15:31:38 CST 2025] <<<<<< START pulsar cluster in normal mode in row_format case >>>>>> + echo 'Waiting for pulsar port to be ready...' Waiting for pulsar port to be ready... + i=0 + nc -z localhost 6650 + /usr/local/pulsar/bin/pulsar standalone --config /tmp/tidb_cdc_test/row_format/pulsar_standalone.conf -nfw --metadata-dir /tmp/tidb_cdc_test/row_format/pulsar-metadata --bookkeeper-dir /tmp/tidb_cdc_test/row_format/pulsar-bookie + i=1 + '[' 1 -gt 20 ']' + sleep 2 find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/consistent_replicate_storage_s3/run.sh using Sink-Type: pulsar... <<================= * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 24927 failed: Connection refused * Failed to connect to 127.0.0.1 port 24927: Connection refused * Closing connection 0 You are running an older version of MinIO released 4 years ago Update: Run `mc admin update` Attempting encryption of all config, IAM users and policies on MinIO backend Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-3484/tiflow-cdc) 4030750720 bytes in 8.35 secs (482565564 bytes/sec) [Pipeline] { [Pipeline] cache Endpoint: http://127.0.0.1:24927 Object API (Amazon S3 compatible): Go: https://docs.min.io/docs/golang-client-quickstart-guide Java: https://docs.min.io/docs/java-client-quickstart-guide Python: https://docs.min.io/docs/python-client-quickstart-guide JavaScript: https://docs.min.io/docs/javascript-client-quickstart-guide .NET: https://docs.min.io/docs/dotnet-client-quickstart-guide + nc -z localhost 6650 + i=2 + '[' 2 -gt 20 ']' + sleep 2 find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/kafka_simple_basic/run.sh using Sink-Type: pulsar... <<================= [Fri Apr 4 15:31:40 CST 2025] <<<<<< run test case kafka_simple_basic success! >>>>>> * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 24927 (#0) > GET / HTTP/1.1 > Host: 127.0.0.1:24927 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 403 Forbidden < Accept-Ranges: bytes < Content-Length: 226 < Content-Security-Policy: block-all-mixed-content < Content-Type: application/xml < Server: MinIO/RELEASE.2020-07-27T18-37-02Z < Vary: Origin < X-Amz-Request-Id: 18330D1D9D73DC40 < X-Xss-Protection: 1; mode=block < Date: Fri, 04 Apr 2025 07:31:41 GMT < { [226 bytes data] * Connection #0 to host 127.0.0.1 left intact Bucket 's3://logbucket/' created [Fri Apr 4 15:31:41 CST 2025] <<<<<< run test case consistent_replicate_storage_s3 success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Exiting on signal: INTERRUPT find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/csv_storage_update_pk_clustered/run.sh using Sink-Type: pulsar... <<================= [Fri Apr 4 15:31:41 CST 2025] <<<<<< run test case csv_storage_update_pk_clustered success! >>>>>> + nc -z localhost 6650 + echo 'Waiting for pulsar namespace to be ready...' Waiting for pulsar namespace to be ready... + i=0 + /usr/local/pulsar/bin/pulsar-admin namespaces list public find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/kafka_simple_basic_avro/run.sh using Sink-Type: pulsar... <<================= [Fri Apr 4 15:31:44 CST 2025] <<<<<< run test case kafka_simple_basic_avro success! >>>>>> public/default =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/consistent_partition_table/run.sh using Sink-Type: pulsar... <<================= [Fri Apr 4 15:31:45 CST 2025] <<<<<< run test case consistent_partition_table success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ++ date + echo '[Fri Apr 4 15:31:45 CST 2025] <<<<<< pulsar is ready >>>>>>' [Fri Apr 4 15:31:45 CST 2025] <<<<<< pulsar is ready >>>>>> + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.row_format.cli.3372.out cli changefeed create --start-ts=457114096602447873 '--sink-uri=pulsar://127.0.0.1:6650/ticdc-row-format-test-27587?protocol=canal-json&enable-tidb-extension=true' find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/csv_storage_update_pk_nonclustered/run.sh using Sink-Type: pulsar... <<================= [Fri Apr 4 15:31:45 CST 2025] <<<<<< run test case csv_storage_update_pk_nonclustered success! >>>>>> Create changefeed successfully! ID: bd9ae331-885e-4ea1-bd2e-3f4da38b3cb1 Info: {"upstream_id":7489357325317402269,"namespace":"default","id":"bd9ae331-885e-4ea1-bd2e-3f4da38b3cb1","sink_uri":"pulsar://127.0.0.1:6650/ticdc-row-format-test-27587?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2025-04-04T15:31:46.16144668+08:00","start_ts":457114096602447873,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"pulsar_config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-10-ga306f4894","resolved_ts":457114096602447873,"checkpoint_ts":457114096602447873,"checkpoint_time":"2025-04-04 15:31:34.388"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/kafka_simple_handle_key_only/run.sh using Sink-Type: pulsar... <<================= [Fri Apr 4 15:31:47 CST 2025] <<<<<< run test case kafka_simple_handle_key_only success! >>>>>> + set +x [Fri Apr 4 15:31:47 CST 2025] <<<<<< START Pulsar consumer in row_format case >>>>>> find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/kafka_big_messages_v2/run.sh using Sink-Type: pulsar... <<================= [Fri Apr 4 15:31:48 CST 2025] <<<<<< run test case kafka_big_messages_v2 success! >>>>>> \033[0;36m<<< Run all test success >>>\033[0m Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-3484/tiflow-cdc) 4030750720 bytes in 8.12 secs (496582134 bytes/sec) [Pipeline] { [Pipeline] cache find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/kafka_simple_handle_key_only_avro/run.sh using Sink-Type: pulsar... <<================= [Fri Apr 4 15:31:51 CST 2025] <<<<<< run test case kafka_simple_handle_key_only_avro success! >>>>>> find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/multi_tables_ddl_v2/run.sh using Sink-Type: pulsar... <<================= [Fri Apr 4 15:31:51 CST 2025] <<<<<< run test case multi_tables_ddl_v2 success! >>>>>> find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/kafka_simple_claim_check/run.sh using Sink-Type: pulsar... <<================= [Fri Apr 4 15:31:54 CST 2025] <<<<<< run test case kafka_simple_claim_check success! >>>>>> find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/multi_topics_v2/run.sh using Sink-Type: pulsar... <<================= [Fri Apr 4 15:31:55 CST 2025] <<<<<< run test case multi_topics_v2 success! >>>>>> table row_format.finish_mark not exists for 1-th check, retry later find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/kafka_simple_claim_check_avro/run.sh using Sink-Type: pulsar... <<================= [Fri Apr 4 15:31:57 CST 2025] <<<<<< run test case kafka_simple_claim_check_avro success! >>>>>> table row_format.finish_mark not exists for 2-th check, retry later find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/storage_cleanup/run.sh using Sink-Type: pulsar... <<================= +++ dirname /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/storage_cleanup/run.sh ++ cd /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/storage_cleanup ++ pwd + CUR=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/storage_cleanup + source /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/storage_cleanup/../_utils/test_prepare ++ UP_TIDB_HOST=127.0.0.1 ++ UP_TIDB_PORT=4000 ++ UP_TIDB_OTHER_PORT=4001 ++ UP_TIDB_STATUS=10080 ++ UP_TIDB_OTHER_STATUS=10081 ++ DOWN_TIDB_HOST=127.0.0.1 ++ DOWN_TIDB_PORT=3306 ++ DOWN_TIDB_STATUS=20080 ++ TLS_TIDB_HOST=127.0.0.1 ++ TLS_TIDB_PORT=3307 ++ TLS_TIDB_STATUS=30080 ++ UP_PD_HOST_1=127.0.0.1 ++ UP_PD_PORT_1=2379 ++ UP_PD_PEER_PORT_1=2380 ++ UP_PD_HOST_2=127.0.0.1 ++ UP_PD_PORT_2=2679 ++ UP_PD_PEER_PORT_2=2680 ++ UP_PD_HOST_3=127.0.0.1 ++ UP_PD_PORT_3=2779 ++ UP_PD_PEER_PORT_3=2780 ++ DOWN_PD_HOST=127.0.0.1 ++ DOWN_PD_PORT=2479 ++ DOWN_PD_PEER_PORT=2480 ++ TLS_PD_HOST=127.0.0.1 ++ TLS_PD_PORT=2579 ++ TLS_PD_PEER_PORT=2580 ++ UP_TIKV_HOST_1=127.0.0.1 ++ UP_TIKV_PORT_1=20160 ++ UP_TIKV_STATUS_PORT_1=20181 ++ UP_TIKV_HOST_2=127.0.0.1 ++ UP_TIKV_PORT_2=20161 ++ UP_TIKV_STATUS_PORT_2=20182 ++ UP_TIKV_HOST_3=127.0.0.1 ++ UP_TIKV_PORT_3=20162 ++ UP_TIKV_STATUS_PORT_3=20183 ++ DOWN_TIKV_HOST=127.0.0.1 ++ DOWN_TIKV_PORT=21160 ++ DOWN_TIKV_STATUS_PORT=21180 ++ TLS_TIKV_HOST=127.0.0.1 ++ TLS_TIKV_PORT=22160 ++ TLS_TIKV_STATUS_PORT=22180 +++ cat /tmp/tidb_cdc_test/KAFKA_VERSION +++ echo 2.4.1 ++ KAFKA_VERSION=2.4.1 + WORK_DIR=/tmp/tidb_cdc_test/storage_cleanup + CDC_BINARY=cdc.test + SINK_TYPE=pulsar + EXIST_FILES=() + CLEANED_FILES=() + trap stop_tidb_cluster EXIT + run pulsar + '[' pulsar '!=' storage ']' + return + check_logs /tmp/tidb_cdc_test/storage_cleanup ++ date + echo '[Fri Apr 4 15:31:58 CST 2025] <<<<<< run test case storage_cleanup success! >>>>>>' [Fri Apr 4 15:31:58 CST 2025] <<<<<< run test case storage_cleanup success! >>>>>> + stop_tidb_cluster Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-3484/tiflow-cdc) 4030750720 bytes in 6.94 secs (581203923 bytes/sec) [Pipeline] { [Pipeline] cache table row_format.finish_mark not exists for 3-th check, retry later find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/canal_json_adapter_compatibility/run.sh using Sink-Type: pulsar... <<================= [Fri Apr 4 15:32:01 CST 2025] <<<<<< run test case canal_json_adapter_compatibility success! >>>>>> find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/csv_storage_basic/run.sh using Sink-Type: pulsar... <<================= [Fri Apr 4 15:32:02 CST 2025] <<<<<< run test case csv_storage_basic success! >>>>>> table row_format.finish_mark not exists for 4-th check, retry later table row_format.finish_mark not exists for 5-th check, retry later find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/canal_json_basic/run.sh using Sink-Type: pulsar... <<================= The 1 times to try to start tidb cluster... find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/csv_storage_multi_tables_ddl/run.sh using Sink-Type: pulsar... <<================= [Fri Apr 4 15:32:05 CST 2025] <<<<<< run test case csv_storage_multi_tables_ddl success! >>>>>> table row_format.finish_mark not exists for 6-th check, retry later table row_format.finish_mark not exists for 7-th check, retry later find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/csv_storage_partition_table/run.sh using Sink-Type: pulsar... <<================= [Fri Apr 4 15:32:08 CST 2025] <<<<<< run test case csv_storage_partition_table success! >>>>>> start tidb cluster in /tmp/tidb_cdc_test/canal_json_basic Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... table row_format.finish_mark not exists for 8-th check, retry later Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-3484/tiflow-cdc) 4030750720 bytes in 7.97 secs (505734675 bytes/sec) [Pipeline] { [Pipeline] cache Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release \033[0;36m<<< Run all test success >>>\033[0m Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table row_format.finish_mark not exists for 9-th check, retry later table row_format.finish_mark not exists for 10-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489357511231917341 TiDB Cluster ID. tikv_gc_leader_uuid 657fedbc4c4000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-0lxq-6vnt9, pid:2434, start at 2025-04-04 15:32:14.28959803 +0800 CST m=+1.601618081 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:34:14.297 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:32:14.257 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:22:14.257 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489357511231917341 TiDB Cluster ID. tikv_gc_leader_uuid 657fedbc4c4000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-0lxq-6vnt9, pid:2434, start at 2025-04-04 15:32:14.28959803 +0800 CST m=+1.601618081 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:34:14.297 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:32:14.257 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:22:14.257 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489357515444193682 TiDB Cluster ID. tikv_gc_leader_uuid 657fedbc5980009 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-0lxq-6vnt9, pid:2509, start at 2025-04-04 15:32:14.323898145 +0800 CST m=+1.567202726 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:34:14.332 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:32:14.310 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:22:14.310 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/canal_json_basic/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/canal_json_basic/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/canal_json_basic/tiflash/log/proxy.log"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/canal_json_basic/tiflash-proxy.toml"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/canal_json_basic/tiflash/db/proxy"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table row_format.finish_mark not exists for 11-th check, retry later [Fri Apr 4 15:32:17 CST 2025] <<<<<< START cdc server in canal_json_basic case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.canal_json_basic.39633965.out server --log-file /tmp/tidb_cdc_test/canal_json_basic/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/canal_json_basic/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table row_format.finish_mark exists check diff successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 4 15:32:20 CST 2025] <<<<<< run test case row_format success! >>>>>> + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 07:32:20 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [817 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a80716e1-ae84-4a17-bf46-5f2f134d62ad {"id":"a80716e1-ae84-4a17-bf46-5f2f134d62ad","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743751937} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffb6dae7ce a80716e1-ae84-4a17-bf46-5f2f134d62ad /tidb/cdc/default/default/upstream/7489357511231917341 {"id":7489357511231917341,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a80716e1-ae84-4a17-bf46-5f2f134d62ad {"id":"a80716e1-ae84-4a17-bf46-5f2f134d62ad","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743751937} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffb6dae7ce a80716e1-ae84-4a17-bf46-5f2f134d62ad /tidb/cdc/default/default/upstream/7489357511231917341 {"id":7489357511231917341,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a80716e1-ae84-4a17-bf46-5f2f134d62ad {"id":"a80716e1-ae84-4a17-bf46-5f2f134d62ad","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743751937} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffb6dae7ce a80716e1-ae84-4a17-bf46-5f2f134d62ad /tidb/cdc/default/default/upstream/7489357511231917341 {"id":7489357511231917341,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x ++ date + echo '[Fri Apr 4 15:32:20 CST 2025] <<<<<< START pulsar cluster in canal_json_basic case >>>>>>' [Fri Apr 4 15:32:20 CST 2025] <<<<<< START pulsar cluster in canal_json_basic case >>>>>> + workdir=/tmp/tidb_cdc_test/canal_json_basic + cluster_type=normal + cd /tmp/tidb_cdc_test/canal_json_basic + DEFAULT_PULSAR_HOME=/usr/local/pulsar + pulsar_dir=/usr/local/pulsar ++ cat + mtls_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderTls brokerClientTlsEnabled=true brokerClientTrustCertsFilePath=/tmp/tidb_cdc_test/canal_json_basic/ca.cert.pem brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls brokerClientAuthenticationParameters={"tlsCertFile":"/tmp/tidb_cdc_test/canal_json_basic/broker_client.cert.pem","tlsKeyFile":"/tmp/tidb_cdc_test/canal_json_basic/broker_client.key-pk8.pem"} brokerServicePortTls=6651 webServicePortTls=8443 tlsTrustCertsFilePath=/tmp/tidb_cdc_test/canal_json_basic/ca.cert.pem tlsCertificateFilePath=/tmp/tidb_cdc_test/canal_json_basic/server.cert.pem tlsKeyFilePath=/tmp/tidb_cdc_test/canal_json_basic/server.key-pk8.pem tlsRequireTrustedClientCertOnConnect=true tlsAllowInsecureConnection=false tlsCertRefreshCheckDurationSec=300' ++ cat + normal_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' ++ cat + mtls_client_conf=' webServiceUrl=https://localhost:8443/ brokerServiceUrl=pulsar+ssl://localhost:6651/ authPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls authParams=tlsCertFile:/tmp/tidb_cdc_test/canal_json_basic/broker_client.cert.pem,tlsKeyFile:/tmp/tidb_cdc_test/canal_json_basic/broker_client.key-pk8.pem tlsTrustCertsFilePath=/tmp/tidb_cdc_test/canal_json_basic/ca.cert.pem' ++ cat + oauth_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/ authPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 authParams={"privateKey":"/tmp/tidb_cdc_test/canal_json_basic/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"}' ++ cat + oauth_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderToken brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 brokerClientAuthenticationParameters={"privateKey":"file:///tmp/tidb_cdc_test/canal_json_basic/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"} tokenSecretKey=data:;base64,U0poWDM2X0thcFlTeWJCdEpxMzVseFhfQnJyNExSVVJTa203UW1YSkdteThwVUZXOUVJT2NWUVBzeWt6OS1qag==' ++ cat + credential_json=' { "client_id":"1234", "client_secret":"e0KVlA2EiBfjoN13olyZd2kv1KL", "audience":"cdc-api-uri", "issuer_url":"http://localhost:9096", "type": "client_credentials" }' ++ cat + cert_server_conf='[ req ] default_bits = 2048 prompt = no default_md = sha256 distinguished_name = dn [ v3_ext ] authorityKeyIdentifier=keyid,issuer:always basicConstraints=CA:FALSE keyUsage=critical, digitalSignature, keyEncipherment extendedKeyUsage=serverAuth subjectAltName=@alt_names [ dn ] CN = server [ alt_names ] DNS.1 = localhost IP.1 = 127.0.0.1' + echo ' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' + cp /usr/local/pulsar/conf/standalone.conf /tmp/tidb_cdc_test/canal_json_basic/pulsar_standalone.conf + pulsar_port=6650 + '[' normal == mtls ']' + '[' normal == oauth ']' + echo 'no cluster type specified, using default configuration.' no cluster type specified, using default configuration. ++ date + echo '[Fri Apr 4 15:32:20 CST 2025] <<<<<< START pulsar cluster in normal mode in canal_json_basic case >>>>>>' [Fri Apr 4 15:32:20 CST 2025] <<<<<< START pulsar cluster in normal mode in canal_json_basic case >>>>>> + echo 'Waiting for pulsar port to be ready...' Waiting for pulsar port to be ready... + i=0 + nc -z localhost 6650 + /usr/local/pulsar/bin/pulsar standalone --config /tmp/tidb_cdc_test/canal_json_basic/pulsar_standalone.conf -nfw --metadata-dir /tmp/tidb_cdc_test/canal_json_basic/pulsar-metadata --bookkeeper-dir /tmp/tidb_cdc_test/canal_json_basic/pulsar-bookie + i=1 + '[' 1 -gt 20 ']' + sleep 2 Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-3484/tiflow-cdc) 4030750720 bytes in 8.40 secs (480016280 bytes/sec) [Pipeline] { [Pipeline] sh [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh pulsar G04 [Pipeline] sh Run cases: foreign_key ddl_puller_lag ddl_only_block_related_table changefeed_auto_stop GIT_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/3484/display/redirect JENKINS_URL=https://do.pingcap.net/jenkins/ EXECUTOR_NUMBER=0 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_pulsar_test","buildid":"1908056220913111045","prowjobid":"b4bfd23f-f137-430c-9c71-ddcc89dd7a6d","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","base_link":"https://github.com/pingcap/tiflow/commit/4c631d5951854ac91765b729fe3849d8e8d5ed8a","pulls":[{"number":12118,"author":"dveeden","sha":"a306f48949e037965a7c1963f500f7e0abbfba51","title":"dm: upgrade go-mysql to v1.12.0","head_ref":"new_gomysql","link":"https://github.com/pingcap/tiflow/pull/12118","commit_link":"https://github.com/pingcap/tiflow/pull/12118/commits/a306f48949e037965a7c1963f500f7e0abbfba51","author_link":"https://github.com/dveeden"}]}} TZ=Asia/Shanghai BUILD_ID=1908056220913111045 POD_LABEL=pingcap_tiflow_pull_cdc_integration_pulsar_test_3484-2bnfb HOSTNAME=pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-2bnf-6v9zm OLDPWD=/home/jenkins RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/3484/display/redirect?page=changes JENKINS_NODE_COOKIE=086ca7dd-7600-448d-a457-99c0831e3e5d GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct JAVA_HOME=/usr/lib/jvm/jre-openjdk JOB_BASE_NAME=pull_cdc_integration_pulsar_test WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test@tmp KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true GIT_URL=https://github.com/PingCAP-QE/ci.git CLASSPATH= KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 FILE_SERVER_URL=http://fileserver.pingcap.net POD_CONTAINER=golang GOPATH=/go BUILD_NUMBER=3484 USE_BAZEL_VERSION=6.5.0 KUBERNETES_PORT=tcp://10.233.0.1:443 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ HOME=/home/jenkins NODE_NAME=pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-2bnf-6v9zm HUDSON_SERVER_COOKIE=83ef27fe9acccc92 JENKINS_HOME=/var/jenkins_home JOB_NAME=pingcap/tiflow/pull_cdc_integration_pulsar_test RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/3484/display/redirect?page=tests KUBERNETES_SERVICE_PORT_HTTPS=443 KUBERNETES_PORT_443_TCP_PORT=443 GIT_PREVIOUS_COMMIT=3e31ec89ba643faa364a78dbc7f062bb55f9c486 HUDSON_HOME=/var/jenkins_home JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/display/redirect PROW_JOB_ID=b4bfd23f-f137-430c-9c71-ddcc89dd7a6d TEST_GROUP=G04 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/3484/display/redirect?page=artifacts BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/3484/ TERM=xterm STAGE_NAME=Test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/ BUILD_DISPLAY_NAME=#3484 SHLVL=5 GIT_BRANCH=origin/main BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-3484 KUBERNETES_SERVICE_PORT=443 NODE_LABELS=pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-2bnf-6v9zm pingcap_tiflow_pull_cdc_integration_pulsar_test_3484-2bnfb PATH=/go/bin:/root/.cargo/bin:/usr/local/go/bin/:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/gradle-7.4.2/bin:/opt/apache-maven-3.8.8/bin:/usr/local/pulsar/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../../scripts/bin GIT_PREVIOUS_SUCCESSFUL_COMMIT=3e31ec89ba643faa364a78dbc7f062bb55f9c486 KUBERNETES_SERVICE_HOST=10.233.0.1 JENKINS_SERVER_COOKIE=durable-1c247575f7a5458f3a8df09baf43395940069f8be33f18a8fa953544ffab3376 _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/foreign_key/run.sh using Sink-Type: pulsar... <<================= The 1 times to try to start tidb cluster... + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh pulsar G06 [Pipeline] sh Run cases: sink_retry changefeed_error ddl_sequence resourcecontrol GIT_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/3484/display/redirect JENKINS_URL=https://do.pingcap.net/jenkins/ EXECUTOR_NUMBER=0 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_pulsar_test","buildid":"1908056220913111045","prowjobid":"b4bfd23f-f137-430c-9c71-ddcc89dd7a6d","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","base_link":"https://github.com/pingcap/tiflow/commit/4c631d5951854ac91765b729fe3849d8e8d5ed8a","pulls":[{"number":12118,"author":"dveeden","sha":"a306f48949e037965a7c1963f500f7e0abbfba51","title":"dm: upgrade go-mysql to v1.12.0","head_ref":"new_gomysql","link":"https://github.com/pingcap/tiflow/pull/12118","commit_link":"https://github.com/pingcap/tiflow/pull/12118/commits/a306f48949e037965a7c1963f500f7e0abbfba51","author_link":"https://github.com/dveeden"}]}} TZ=Asia/Shanghai BUILD_ID=1908056220913111045 POD_LABEL=pingcap_tiflow_pull_cdc_integration_pulsar_test_3484-4ttz8 HOSTNAME=pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-4ttz-989qr OLDPWD=/home/jenkins RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/3484/display/redirect?page=changes JENKINS_NODE_COOKIE=d9f8646d-b1f9-449e-814a-6c15b667d21c GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct JAVA_HOME=/usr/lib/jvm/jre-openjdk JOB_BASE_NAME=pull_cdc_integration_pulsar_test WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test@tmp KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true GIT_URL=https://github.com/PingCAP-QE/ci.git CLASSPATH= KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 FILE_SERVER_URL=http://fileserver.pingcap.net POD_CONTAINER=golang GOPATH=/go BUILD_NUMBER=3484 USE_BAZEL_VERSION=6.5.0 KUBERNETES_PORT=tcp://10.233.0.1:443 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ HOME=/home/jenkins NODE_NAME=pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-4ttz-989qr HUDSON_SERVER_COOKIE=83ef27fe9acccc92 JENKINS_HOME=/var/jenkins_home JOB_NAME=pingcap/tiflow/pull_cdc_integration_pulsar_test RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/3484/display/redirect?page=tests KUBERNETES_SERVICE_PORT_HTTPS=443 KUBERNETES_PORT_443_TCP_PORT=443 GIT_PREVIOUS_COMMIT=3e31ec89ba643faa364a78dbc7f062bb55f9c486 HUDSON_HOME=/var/jenkins_home JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/display/redirect PROW_JOB_ID=b4bfd23f-f137-430c-9c71-ddcc89dd7a6d TEST_GROUP=G06 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/3484/display/redirect?page=artifacts BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/3484/ TERM=xterm STAGE_NAME=Test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/ BUILD_DISPLAY_NAME=#3484 SHLVL=5 GIT_BRANCH=origin/main BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-3484 KUBERNETES_SERVICE_PORT=443 NODE_LABELS=pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-4ttz-989qr pingcap_tiflow_pull_cdc_integration_pulsar_test_3484-4ttz8 PATH=/go/bin:/root/.cargo/bin:/usr/local/go/bin/:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/gradle-7.4.2/bin:/opt/apache-maven-3.8.8/bin:/usr/local/pulsar/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../../scripts/bin GIT_PREVIOUS_SUCCESSFUL_COMMIT=3e31ec89ba643faa364a78dbc7f062bb55f9c486 KUBERNETES_SERVICE_HOST=10.233.0.1 JENKINS_SERVER_COOKIE=durable-1c247575f7a5458f3a8df09baf43395940069f8be33f18a8fa953544ffab3376 _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/sink_retry/run.sh using Sink-Type: pulsar... <<================= The 1 times to try to start tidb cluster... + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh pulsar G05 [Pipeline] sh Run cases: charset_gbk ddl_manager multi_source vector GIT_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/3484/display/redirect JENKINS_URL=https://do.pingcap.net/jenkins/ EXECUTOR_NUMBER=0 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_pulsar_test","buildid":"1908056220913111045","prowjobid":"b4bfd23f-f137-430c-9c71-ddcc89dd7a6d","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","base_link":"https://github.com/pingcap/tiflow/commit/4c631d5951854ac91765b729fe3849d8e8d5ed8a","pulls":[{"number":12118,"author":"dveeden","sha":"a306f48949e037965a7c1963f500f7e0abbfba51","title":"dm: upgrade go-mysql to v1.12.0","head_ref":"new_gomysql","link":"https://github.com/pingcap/tiflow/pull/12118","commit_link":"https://github.com/pingcap/tiflow/pull/12118/commits/a306f48949e037965a7c1963f500f7e0abbfba51","author_link":"https://github.com/dveeden"}]}} TZ=Asia/Shanghai BUILD_ID=1908056220913111045 POD_LABEL=pingcap_tiflow_pull_cdc_integration_pulsar_test_3484-sfgg7 HOSTNAME=pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-sfgg-lkh0k OLDPWD=/home/jenkins RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/3484/display/redirect?page=changes JENKINS_NODE_COOKIE=fd2958ed-0ef3-428a-9bcb-5e601b239de5 GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct JAVA_HOME=/usr/lib/jvm/jre-openjdk JOB_BASE_NAME=pull_cdc_integration_pulsar_test WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test@tmp KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true GIT_URL=https://github.com/PingCAP-QE/ci.git CLASSPATH= KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 FILE_SERVER_URL=http://fileserver.pingcap.net POD_CONTAINER=golang GOPATH=/go BUILD_NUMBER=3484 USE_BAZEL_VERSION=6.5.0 KUBERNETES_PORT=tcp://10.233.0.1:443 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ HOME=/home/jenkins NODE_NAME=pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-sfgg-lkh0k HUDSON_SERVER_COOKIE=83ef27fe9acccc92 JENKINS_HOME=/var/jenkins_home JOB_NAME=pingcap/tiflow/pull_cdc_integration_pulsar_test RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/3484/display/redirect?page=tests KUBERNETES_SERVICE_PORT_HTTPS=443 KUBERNETES_PORT_443_TCP_PORT=443 GIT_PREVIOUS_COMMIT=3e31ec89ba643faa364a78dbc7f062bb55f9c486 HUDSON_HOME=/var/jenkins_home JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/display/redirect PROW_JOB_ID=b4bfd23f-f137-430c-9c71-ddcc89dd7a6d TEST_GROUP=G05 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/3484/display/redirect?page=artifacts BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/3484/ TERM=xterm STAGE_NAME=Test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/ BUILD_DISPLAY_NAME=#3484 SHLVL=5 GIT_BRANCH=origin/main BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-3484 KUBERNETES_SERVICE_PORT=443 NODE_LABELS=pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-sfgg-lkh0k pingcap_tiflow_pull_cdc_integration_pulsar_test_3484-sfgg7 PATH=/go/bin:/root/.cargo/bin:/usr/local/go/bin/:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/gradle-7.4.2/bin:/opt/apache-maven-3.8.8/bin:/usr/local/pulsar/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../../scripts/bin GIT_PREVIOUS_SUCCESSFUL_COMMIT=3e31ec89ba643faa364a78dbc7f062bb55f9c486 KUBERNETES_SERVICE_HOST=10.233.0.1 JENKINS_SERVER_COOKIE=durable-1c247575f7a5458f3a8df09baf43395940069f8be33f18a8fa953544ffab3376 _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/charset_gbk/run.sh using Sink-Type: pulsar... <<================= The 1 times to try to start tidb cluster... + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh pulsar G08 [Pipeline] sh + nc -z localhost 6650 + i=2 + '[' 2 -gt 20 ']' + sleep 2 Run cases: processor_err_chan changefeed_reconstruct multi_capture synced_status_with_redo GIT_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/3484/display/redirect JENKINS_URL=https://do.pingcap.net/jenkins/ EXECUTOR_NUMBER=0 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_pulsar_test","buildid":"1908056220913111045","prowjobid":"b4bfd23f-f137-430c-9c71-ddcc89dd7a6d","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","base_link":"https://github.com/pingcap/tiflow/commit/4c631d5951854ac91765b729fe3849d8e8d5ed8a","pulls":[{"number":12118,"author":"dveeden","sha":"a306f48949e037965a7c1963f500f7e0abbfba51","title":"dm: upgrade go-mysql to v1.12.0","head_ref":"new_gomysql","link":"https://github.com/pingcap/tiflow/pull/12118","commit_link":"https://github.com/pingcap/tiflow/pull/12118/commits/a306f48949e037965a7c1963f500f7e0abbfba51","author_link":"https://github.com/dveeden"}]}} TZ=Asia/Shanghai BUILD_ID=1908056220913111045 POD_LABEL=pingcap_tiflow_pull_cdc_integration_pulsar_test_3484-9bldc HOSTNAME=pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-9bld-w6hq9 OLDPWD=/home/jenkins RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/3484/display/redirect?page=changes JENKINS_NODE_COOKIE=478999b7-23fc-4d68-9f8a-0b35c0a8cefa GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct JAVA_HOME=/usr/lib/jvm/jre-openjdk JOB_BASE_NAME=pull_cdc_integration_pulsar_test WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test@tmp KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true GIT_URL=https://github.com/PingCAP-QE/ci.git CLASSPATH= KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 FILE_SERVER_URL=http://fileserver.pingcap.net POD_CONTAINER=golang GOPATH=/go BUILD_NUMBER=3484 USE_BAZEL_VERSION=6.5.0 KUBERNETES_PORT=tcp://10.233.0.1:443 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ HOME=/home/jenkins NODE_NAME=pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-9bld-w6hq9 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 JENKINS_HOME=/var/jenkins_home JOB_NAME=pingcap/tiflow/pull_cdc_integration_pulsar_test RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/3484/display/redirect?page=tests KUBERNETES_SERVICE_PORT_HTTPS=443 KUBERNETES_PORT_443_TCP_PORT=443 GIT_PREVIOUS_COMMIT=3e31ec89ba643faa364a78dbc7f062bb55f9c486 HUDSON_HOME=/var/jenkins_home JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/display/redirect PROW_JOB_ID=b4bfd23f-f137-430c-9c71-ddcc89dd7a6d TEST_GROUP=G08 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/3484/display/redirect?page=artifacts BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/3484/ TERM=xterm STAGE_NAME=Test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/ BUILD_DISPLAY_NAME=#3484 SHLVL=5 GIT_BRANCH=origin/main BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-3484 KUBERNETES_SERVICE_PORT=443 NODE_LABELS=pingcap_tiflow_pull_cdc_integration_pulsar_test_3484-9bldc pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-9bld-w6hq9 PATH=/go/bin:/root/.cargo/bin:/usr/local/go/bin/:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/gradle-7.4.2/bin:/opt/apache-maven-3.8.8/bin:/usr/local/pulsar/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../../scripts/bin GIT_PREVIOUS_SUCCESSFUL_COMMIT=3e31ec89ba643faa364a78dbc7f062bb55f9c486 KUBERNETES_SERVICE_HOST=10.233.0.1 JENKINS_SERVER_COOKIE=durable-1c247575f7a5458f3a8df09baf43395940069f8be33f18a8fa953544ffab3376 _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/processor_err_chan/run.sh using Sink-Type: pulsar... <<================= The 1 times to try to start tidb cluster... + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh pulsar G09 Run cases: gc_safepoint changefeed_pause_resume cli_with_auth savepoint synced_status GIT_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/3484/display/redirect JENKINS_URL=https://do.pingcap.net/jenkins/ EXECUTOR_NUMBER=0 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_pulsar_test","buildid":"1908056220913111045","prowjobid":"b4bfd23f-f137-430c-9c71-ddcc89dd7a6d","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","base_link":"https://github.com/pingcap/tiflow/commit/4c631d5951854ac91765b729fe3849d8e8d5ed8a","pulls":[{"number":12118,"author":"dveeden","sha":"a306f48949e037965a7c1963f500f7e0abbfba51","title":"dm: upgrade go-mysql to v1.12.0","head_ref":"new_gomysql","link":"https://github.com/pingcap/tiflow/pull/12118","commit_link":"https://github.com/pingcap/tiflow/pull/12118/commits/a306f48949e037965a7c1963f500f7e0abbfba51","author_link":"https://github.com/dveeden"}]}} TZ=Asia/Shanghai BUILD_ID=1908056220913111045 POD_LABEL=pingcap_tiflow_pull_cdc_integration_pulsar_test_3484-qg8gt HOSTNAME=pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-qg8g-pl0pp OLDPWD=/home/jenkins RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/3484/display/redirect?page=changes JENKINS_NODE_COOKIE=aca348e4-7485-4950-b9e5-b5e520af4ae7 GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct JAVA_HOME=/usr/lib/jvm/jre-openjdk JOB_BASE_NAME=pull_cdc_integration_pulsar_test WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test@tmp KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true GIT_URL=https://github.com/PingCAP-QE/ci.git CLASSPATH= KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 FILE_SERVER_URL=http://fileserver.pingcap.net POD_CONTAINER=golang GOPATH=/go BUILD_NUMBER=3484 USE_BAZEL_VERSION=6.5.0 KUBERNETES_PORT=tcp://10.233.0.1:443 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ HOME=/home/jenkins NODE_NAME=pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-qg8g-pl0pp HUDSON_SERVER_COOKIE=83ef27fe9acccc92 JENKINS_HOME=/var/jenkins_home JOB_NAME=pingcap/tiflow/pull_cdc_integration_pulsar_test RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/3484/display/redirect?page=tests KUBERNETES_SERVICE_PORT_HTTPS=443 KUBERNETES_PORT_443_TCP_PORT=443 GIT_PREVIOUS_COMMIT=3e31ec89ba643faa364a78dbc7f062bb55f9c486 HUDSON_HOME=/var/jenkins_home JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/display/redirect PROW_JOB_ID=b4bfd23f-f137-430c-9c71-ddcc89dd7a6d TEST_GROUP=G09 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/3484/display/redirect?page=artifacts BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/3484/ TERM=xterm STAGE_NAME=Test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/ BUILD_DISPLAY_NAME=#3484 SHLVL=5 GIT_BRANCH=origin/main BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-3484 KUBERNETES_SERVICE_PORT=443 NODE_LABELS=pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-qg8g-pl0pp pingcap_tiflow_pull_cdc_integration_pulsar_test_3484-qg8gt PATH=/go/bin:/root/.cargo/bin:/usr/local/go/bin/:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/gradle-7.4.2/bin:/opt/apache-maven-3.8.8/bin:/usr/local/pulsar/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../../scripts/bin GIT_PREVIOUS_SUCCESSFUL_COMMIT=3e31ec89ba643faa364a78dbc7f062bb55f9c486 KUBERNETES_SERVICE_HOST=10.233.0.1 JENKINS_SERVER_COOKIE=durable-1c247575f7a5458f3a8df09baf43395940069f8be33f18a8fa953544ffab3376 _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/gc_safepoint/run.sh using Sink-Type: pulsar... <<================= The 1 times to try to start tidb cluster... + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh pulsar G07 Run cases: kv_client_stream_reconnect cdc split_region GIT_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/3484/display/redirect JENKINS_URL=https://do.pingcap.net/jenkins/ EXECUTOR_NUMBER=0 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_pulsar_test","buildid":"1908056220913111045","prowjobid":"b4bfd23f-f137-430c-9c71-ddcc89dd7a6d","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","base_link":"https://github.com/pingcap/tiflow/commit/4c631d5951854ac91765b729fe3849d8e8d5ed8a","pulls":[{"number":12118,"author":"dveeden","sha":"a306f48949e037965a7c1963f500f7e0abbfba51","title":"dm: upgrade go-mysql to v1.12.0","head_ref":"new_gomysql","link":"https://github.com/pingcap/tiflow/pull/12118","commit_link":"https://github.com/pingcap/tiflow/pull/12118/commits/a306f48949e037965a7c1963f500f7e0abbfba51","author_link":"https://github.com/dveeden"}]}} TZ=Asia/Shanghai BUILD_ID=1908056220913111045 POD_LABEL=pingcap_tiflow_pull_cdc_integration_pulsar_test_3484-jd09w HOSTNAME=pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-jd09-mfgkq OLDPWD=/home/jenkins RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/3484/display/redirect?page=changes JENKINS_NODE_COOKIE=91d3dccb-7601-4b54-aadb-0fbd18c6ddfc GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct JAVA_HOME=/usr/lib/jvm/jre-openjdk JOB_BASE_NAME=pull_cdc_integration_pulsar_test WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test@tmp KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true GIT_URL=https://github.com/PingCAP-QE/ci.git CLASSPATH= KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 FILE_SERVER_URL=http://fileserver.pingcap.net POD_CONTAINER=golang GOPATH=/go BUILD_NUMBER=3484 USE_BAZEL_VERSION=6.5.0 KUBERNETES_PORT=tcp://10.233.0.1:443 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ HOME=/home/jenkins NODE_NAME=pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-jd09-mfgkq HUDSON_SERVER_COOKIE=83ef27fe9acccc92 JENKINS_HOME=/var/jenkins_home JOB_NAME=pingcap/tiflow/pull_cdc_integration_pulsar_test RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/3484/display/redirect?page=tests KUBERNETES_SERVICE_PORT_HTTPS=443 KUBERNETES_PORT_443_TCP_PORT=443 GIT_PREVIOUS_COMMIT=3e31ec89ba643faa364a78dbc7f062bb55f9c486 HUDSON_HOME=/var/jenkins_home JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/display/redirect PROW_JOB_ID=b4bfd23f-f137-430c-9c71-ddcc89dd7a6d TEST_GROUP=G07 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/3484/display/redirect?page=artifacts BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/3484/ TERM=xterm STAGE_NAME=Test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/ BUILD_DISPLAY_NAME=#3484 SHLVL=5 GIT_BRANCH=origin/main BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-3484 KUBERNETES_SERVICE_PORT=443 NODE_LABELS=pingcap_tiflow_pull_cdc_integration_pulsar_test_3484-jd09w pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-jd09-mfgkq PATH=/go/bin:/root/.cargo/bin:/usr/local/go/bin/:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/gradle-7.4.2/bin:/opt/apache-maven-3.8.8/bin:/usr/local/pulsar/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../../scripts/bin GIT_PREVIOUS_SUCCESSFUL_COMMIT=3e31ec89ba643faa364a78dbc7f062bb55f9c486 KUBERNETES_SERVICE_HOST=10.233.0.1 JENKINS_SERVER_COOKIE=durable-1c247575f7a5458f3a8df09baf43395940069f8be33f18a8fa953544ffab3376 _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/kv_client_stream_reconnect/run.sh using Sink-Type: pulsar... <<================= The 1 times to try to start tidb cluster... [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-3484/tiflow-cdc already exists) [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-3484/tiflow-cdc already exists) [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv + nc -z localhost 6650 + echo 'Waiting for pulsar namespace to be ready...' Waiting for pulsar namespace to be ready... + i=0 + /usr/local/pulsar/bin/pulsar-admin namespaces list public [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { start tidb cluster in /tmp/tidb_cdc_test/sink_retry Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { start tidb cluster in /tmp/tidb_cdc_test/charset_gbk Starting Upstream PD... [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/foreign_key Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... [Pipeline] // cache [Pipeline] } [Pipeline] // cache [Pipeline] } [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage start tidb cluster in /tmp/tidb_cdc_test/gc_safepoint Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { start tidb cluster in /tmp/tidb_cdc_test/processor_err_chan Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... Tenant not found Reason: Tenant not found [Pipeline] // dir [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow [Pipeline] { + i=1 + '[' 1 -gt 20 ']' + sleep 2 [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow [Pipeline] { Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release [Pipeline] cache start tidb cluster in /tmp/tidb_cdc_test/kv_client_stream_reconnect Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... Verifying downstream PD is started... Verifying downstream PD is started... Verifying downstream PD is started... Verifying downstream PD is started... + /usr/local/pulsar/bin/pulsar-admin namespaces list public Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/drop_many_tables/run.sh using Sink-Type: pulsar... <<================= The 1 times to try to start tidb cluster... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release public/default Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ++ date + echo '[Fri Apr 4 15:32:30 CST 2025] <<<<<< pulsar is ready >>>>>>' [Fri Apr 4 15:32:30 CST 2025] <<<<<< pulsar is ready >>>>>> + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.canal_json_basic.cli.4526.out cli changefeed create '--sink-uri=pulsar://127.0.0.1:6650/ticdc-canal-json-basic?protocol=canal-json&enable-tidb-extension=true' Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489357576914434452 TiDB Cluster ID. tikv_gc_leader_uuid 657fedcbee4000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-4ttz-989qr, pid:1345, start at 2025-04-04 15:32:30.300146036 +0800 CST m=+1.425244094 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:34:30.307 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:32:30.265 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:22:30.265 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489357576914434452 TiDB Cluster ID. tikv_gc_leader_uuid 657fedcbee4000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-4ttz-989qr, pid:1345, start at 2025-04-04 15:32:30.300146036 +0800 CST m=+1.425244094 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:34:30.307 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:32:30.265 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:22:30.265 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489357580621406483 TiDB Cluster ID. tikv_gc_leader_uuid 657fedcc1ec000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-4ttz-989qr, pid:1432, start at 2025-04-04 15:32:30.489946515 +0800 CST m=+1.546832720 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:34:30.498 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:32:30.459 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:22:30.459 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/sink_retry/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/sink_retry/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/sink_retry/tiflash-proxy.toml"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/sink_retry/tiflash/db/proxy"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/sink_retry/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Create changefeed successfully! ID: 3140a949-bd68-4c6b-a7cd-281e5db30993 Info: {"upstream_id":7489357511231917341,"namespace":"default","id":"3140a949-bd68-4c6b-a7cd-281e5db30993","sink_uri":"pulsar://127.0.0.1:6650/ticdc-canal-json-basic?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2025-04-04T15:32:31.916571537+08:00","start_ts":457114111523422213,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"pulsar_config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-10-ga306f4894","resolved_ts":457114111523422213,"checkpoint_ts":457114111523422213,"checkpoint_time":"2025-04-04 15:32:31.307"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x start tidb cluster in /tmp/tidb_cdc_test/drop_many_tables Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sink_retry.cli.2863.out cli tso query --pd=http://127.0.0.1:2379 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489357587103125569 TiDB Cluster ID. tikv_gc_leader_uuid 657fedcddfc0013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-2bnf-6v9zm, pid:1495, start at 2025-04-04 15:32:32.300685922 +0800 CST m=+1.414686845 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:34:32.308 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:32:32.305 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:22:32.305 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489357587103125569 TiDB Cluster ID. tikv_gc_leader_uuid 657fedcddfc0013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-2bnf-6v9zm, pid:1495, start at 2025-04-04 15:32:32.300685922 +0800 CST m=+1.414686845 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:34:32.308 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:32:32.305 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:22:32.305 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489357588176224844 TiDB Cluster ID. tikv_gc_leader_uuid 657fedce0680002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-2bnf-6v9zm, pid:1573, start at 2025-04-04 15:32:32.410372205 +0800 CST m=+1.446079355 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:34:32.417 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:32:32.410 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:22:32.410 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/foreign_key/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/foreign_key/tiflash/log/error.log arg matches is ArgMatches { args: {"addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/foreign_key/tiflash-proxy.toml"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/foreign_key/tiflash/db/proxy"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/foreign_key/tiflash/log/proxy.log"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489357581164055148 TiDB Cluster ID. tikv_gc_leader_uuid 657fedce678000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-sfgg-lkh0k, pid:1424, start at 2025-04-04 15:32:32.828283911 +0800 CST m=+1.518197268 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:34:32.836 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:32:32.798 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:22:32.798 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489357581164055148 TiDB Cluster ID. tikv_gc_leader_uuid 657fedce678000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-sfgg-lkh0k, pid:1424, start at 2025-04-04 15:32:32.828283911 +0800 CST m=+1.518197268 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:34:32.836 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:32:32.798 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:22:32.798 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489357589478587477 TiDB Cluster ID. tikv_gc_leader_uuid 657fedce74c000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-sfgg-lkh0k, pid:1488, start at 2025-04-04 15:32:32.892500761 +0800 CST m=+1.508505853 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:34:32.900 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:32:32.901 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:22:32.901 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/charset_gbk/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/charset_gbk/tiflash/log/error.log arg matches is ArgMatches { args: {"log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/charset_gbk/tiflash/log/proxy.log"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/charset_gbk/tiflash-proxy.toml"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/charset_gbk/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489357587574651632 TiDB Cluster ID. tikv_gc_leader_uuid 657fedcf594000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-jd09-mfgkq, pid:1481, start at 2025-04-04 15:32:33.784949264 +0800 CST m=+1.559688446 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:34:33.792 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:32:33.765 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:22:33.765 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489357587574651632 TiDB Cluster ID. tikv_gc_leader_uuid 657fedcf594000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-jd09-mfgkq, pid:1481, start at 2025-04-04 15:32:33.784949264 +0800 CST m=+1.559688446 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:34:33.792 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:32:33.765 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:22:33.765 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489357595300874442 TiDB Cluster ID. tikv_gc_leader_uuid 657fedcf650000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-jd09-mfgkq, pid:1560, start at 2025-04-04 15:32:33.829675052 +0800 CST m=+1.535313491 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:34:33.838 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:32:33.812 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:22:33.812 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489357587702590182 TiDB Cluster ID. tikv_gc_leader_uuid 657fedcf784000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-qg8g-pl0pp, pid:1486, start at 2025-04-04 15:32:33.92448 +0800 CST m=+1.556539657 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:34:33.932 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:32:33.889 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:22:33.889 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489357587702590182 TiDB Cluster ID. tikv_gc_leader_uuid 657fedcf784000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-qg8g-pl0pp, pid:1486, start at 2025-04-04 15:32:33.92448 +0800 CST m=+1.556539657 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:34:33.932 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:32:33.889 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:22:33.889 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489357595795544940 TiDB Cluster ID. tikv_gc_leader_uuid 657fedcf7c80006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-qg8g-pl0pp, pid:1561, start at 2025-04-04 15:32:33.916205376 +0800 CST m=+1.468991624 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:34:33.927 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:32:33.906 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:22:33.906 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/kv_client_stream_reconnect/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/kv_client_stream_reconnect/tiflash/log/error.log arg matches is ArgMatches { args: {"pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/kv_client_stream_reconnect/tiflash/db/proxy"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/kv_client_stream_reconnect/tiflash/log/proxy.log"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/kv_client_stream_reconnect/tiflash-proxy.toml"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489357587056082243 TiDB Cluster ID. UTC Build Time: 2025-04-03 03:49:11 tikv_gc_leader_uuid 657fedcf208000d Current GC worker leader UUID. (DO NOT EDIT) Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-9bld-w6hq9, pid:1410, start at 2025-04-04 15:32:33.574679302 +0800 CST m=+1.432087097 Host name and pid of current GC leader. (DO NOT EDIT) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release tikv_gc_leader_lease 20250404-15:34:33.582 +0800 Current GC worker leader lease. (DO NOT EDIT) Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/gc_safepoint/tiflash/log/server.log tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. Logging errors to /tmp/tidb_cdc_test/gc_safepoint/tiflash/log/error.log tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:32:33.587 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:22:33.587 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489357587056082243 TiDB Cluster ID. tikv_gc_leader_uuid 657fedcf208000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-9bld-w6hq9, pid:1410, start at 2025-04-04 15:32:33.574679302 +0800 CST m=+1.432087097 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:34:33.582 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:32:33.587 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:22:33.587 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489357594423797105 TiDB Cluster ID. tikv_gc_leader_uuid 657fedcf388000d Current GC worker leader UUID. (DO NOT EDIT) arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/gc_safepoint/tiflash/db/proxy"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/gc_safepoint/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/gc_safepoint/tiflash/log/proxy.log"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-9bld-w6hq9, pid:1484, start at 2025-04-04 15:32:33.683536901 +0800 CST m=+1.463191330 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:34:33.691 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:32:33.685 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:22:33.685 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/processor_err_chan/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/processor_err_chan/tiflash/log/error.log arg matches is ArgMatches { args: {"memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/processor_err_chan/tiflash/db/proxy"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/processor_err_chan/tiflash-proxy.toml"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/processor_err_chan/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + set +x + tso='457114112115081217 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 457114112115081217 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x ***************** properties ***************** "readallfields"="true" "insertproportion"="0" "threadcount"="2" "updateproportion"="0" "readproportion"="0" "mysql.host"="127.0.0.1" "operationcount"="0" "dotransactions"="false" "scanproportion"="0" "mysql.user"="root" "mysql.db"="sink_retry" "workload"="core" "recordcount"="10" "requestdistribution"="uniform" "mysql.port"="4000" ********************************************** Run finished, takes 7.924273ms INSERT - Takes(s): 0.0, Count: 10, OPS: 2065.3, Avg(us): 1509, Min(us): 917, Max(us): 2997, 95th(us): 3000, 99th(us): 3000 [Fri Apr 4 15:32:35 CST 2025] <<<<<< START cdc server in sink_retry case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sink/dmlsink/txn/mysql/MySQLSinkTxnRandomError=25%return(true)' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sink_retry.29212923.out server --log-file /tmp/tidb_cdc_test/sink_retry/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/sink_retry/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.foreign_key.cli.2881.out cli tso query --pd=http://127.0.0.1:2379 [Fri Apr 4 15:32:35 CST 2025] <<<<<< START cdc server in charset_gbk case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.charset_gbk.28772879.out server --log-file /tmp/tidb_cdc_test/charset_gbk/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/charset_gbk/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ++ date + echo '[Fri Apr 4 15:32:36 CST 2025] <<<<<< START pulsar cluster in kv_client_stream_reconnect case >>>>>>' [Fri Apr 4 15:32:36 CST 2025] <<<<<< START pulsar cluster in kv_client_stream_reconnect case >>>>>> + workdir=/tmp/tidb_cdc_test/kv_client_stream_reconnect + cluster_type=oauth + cd /tmp/tidb_cdc_test/kv_client_stream_reconnect + DEFAULT_PULSAR_HOME=/usr/local/pulsar + pulsar_dir=/usr/local/pulsar ++ cat + mtls_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderTls brokerClientTlsEnabled=true brokerClientTrustCertsFilePath=/tmp/tidb_cdc_test/kv_client_stream_reconnect/ca.cert.pem brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls brokerClientAuthenticationParameters={"tlsCertFile":"/tmp/tidb_cdc_test/kv_client_stream_reconnect/broker_client.cert.pem","tlsKeyFile":"/tmp/tidb_cdc_test/kv_client_stream_reconnect/broker_client.key-pk8.pem"} brokerServicePortTls=6651 webServicePortTls=8443 tlsTrustCertsFilePath=/tmp/tidb_cdc_test/kv_client_stream_reconnect/ca.cert.pem tlsCertificateFilePath=/tmp/tidb_cdc_test/kv_client_stream_reconnect/server.cert.pem tlsKeyFilePath=/tmp/tidb_cdc_test/kv_client_stream_reconnect/server.key-pk8.pem tlsRequireTrustedClientCertOnConnect=true tlsAllowInsecureConnection=false tlsCertRefreshCheckDurationSec=300' ++ cat + normal_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' ++ cat + mtls_client_conf=' webServiceUrl=https://localhost:8443/ brokerServiceUrl=pulsar+ssl://localhost:6651/ authPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls authParams=tlsCertFile:/tmp/tidb_cdc_test/kv_client_stream_reconnect/broker_client.cert.pem,tlsKeyFile:/tmp/tidb_cdc_test/kv_client_stream_reconnect/broker_client.key-pk8.pem tlsTrustCertsFilePath=/tmp/tidb_cdc_test/kv_client_stream_reconnect/ca.cert.pem' Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ++ date + echo '[Fri Apr 4 15:32:36 CST 2025] <<<<<< START pulsar cluster in processor_err_chan case >>>>>>' [Fri Apr 4 15:32:36 CST 2025] <<<<<< START pulsar cluster in processor_err_chan case >>>>>> + workdir=/tmp/tidb_cdc_test/processor_err_chan + cluster_type=normal + cd /tmp/tidb_cdc_test/processor_err_chan + DEFAULT_PULSAR_HOME=/usr/local/pulsar + pulsar_dir=/usr/local/pulsar ++ cat + mtls_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderTls brokerClientTlsEnabled=true brokerClientTrustCertsFilePath=/tmp/tidb_cdc_test/processor_err_chan/ca.cert.pem brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls brokerClientAuthenticationParameters={"tlsCertFile":"/tmp/tidb_cdc_test/processor_err_chan/broker_client.cert.pem","tlsKeyFile":"/tmp/tidb_cdc_test/processor_err_chan/broker_client.key-pk8.pem"} brokerServicePortTls=6651 webServicePortTls=8443 tlsTrustCertsFilePath=/tmp/tidb_cdc_test/processor_err_chan/ca.cert.pem tlsCertificateFilePath=/tmp/tidb_cdc_test/processor_err_chan/server.cert.pem tlsKeyFilePath=/tmp/tidb_cdc_test/processor_err_chan/server.key-pk8.pem tlsRequireTrustedClientCertOnConnect=true tlsAllowInsecureConnection=false tlsCertRefreshCheckDurationSec=300' ++ cat + normal_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' ++ cat + mtls_client_conf=' webServiceUrl=https://localhost:8443/ brokerServiceUrl=pulsar+ssl://localhost:6651/ authPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls authParams=tlsCertFile:/tmp/tidb_cdc_test/processor_err_chan/broker_client.cert.pem,tlsKeyFile:/tmp/tidb_cdc_test/processor_err_chan/broker_client.key-pk8.pem tlsTrustCertsFilePath=/tmp/tidb_cdc_test/processor_err_chan/ca.cert.pem' ++ cat + oauth_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/ authPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 authParams={"privateKey":"/tmp/tidb_cdc_test/processor_err_chan/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"}' ++ cat + oauth_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderToken brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 brokerClientAuthenticationParameters={"privateKey":"file:///tmp/tidb_cdc_test/processor_err_chan/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"} tokenSecretKey=data:;base64,U0poWDM2X0thcFlTeWJCdEpxMzVseFhfQnJyNExSVVJTa203UW1YSkdteThwVUZXOUVJT2NWUVBzeWt6OS1qag==' ++ cat + credential_json=' { "client_id":"1234", "client_secret":"e0KVlA2EiBfjoN13olyZd2kv1KL", "audience":"cdc-api-uri", "issuer_url":"http://localhost:9096", "type": "client_credentials" }' ++ cat + cert_server_conf='[ req ] default_bits = 2048 prompt = no default_md = sha256 distinguished_name = dn [ v3_ext ] authorityKeyIdentifier=keyid,issuer:always basicConstraints=CA:FALSE keyUsage=critical, digitalSignature, keyEncipherment extendedKeyUsage=serverAuth subjectAltName=@alt_names [ dn ] CN = server [ alt_names ] DNS.1 = localhost IP.1 = 127.0.0.1' + echo ' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' + cp /usr/local/pulsar/conf/standalone.conf /tmp/tidb_cdc_test/processor_err_chan/pulsar_standalone.conf + pulsar_port=6650 + '[' normal == mtls ']' + '[' normal == oauth ']' + echo 'no cluster type specified, using default configuration.' no cluster type specified, using default configuration. ++ date + echo '[Fri Apr 4 15:32:36 CST 2025] <<<<<< START pulsar cluster in normal mode in processor_err_chan case >>>>>>' [Fri Apr 4 15:32:36 CST 2025] <<<<<< START pulsar cluster in normal mode in processor_err_chan case >>>>>> + echo 'Waiting for pulsar port to be ready...' Waiting for pulsar port to be ready... + i=0 + nc -z localhost 6650 + /usr/local/pulsar/bin/pulsar standalone --config /tmp/tidb_cdc_test/processor_err_chan/pulsar_standalone.conf -nfw --metadata-dir /tmp/tidb_cdc_test/processor_err_chan/pulsar-metadata --bookkeeper-dir /tmp/tidb_cdc_test/processor_err_chan/pulsar-bookie + i=1 + '[' 1 -gt 20 ']' + sleep 2 ++ cat + oauth_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/ authPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 authParams={"privateKey":"/tmp/tidb_cdc_test/kv_client_stream_reconnect/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"}' ++ cat + oauth_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderToken brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 brokerClientAuthenticationParameters={"privateKey":"file:///tmp/tidb_cdc_test/kv_client_stream_reconnect/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"} tokenSecretKey=data:;base64,U0poWDM2X0thcFlTeWJCdEpxMzVseFhfQnJyNExSVVJTa203UW1YSkdteThwVUZXOUVJT2NWUVBzeWt6OS1qag==' ++ cat + credential_json=' { "client_id":"1234", "client_secret":"e0KVlA2EiBfjoN13olyZd2kv1KL", "audience":"cdc-api-uri", "issuer_url":"http://localhost:9096", "type": "client_credentials" }' ++ cat + cert_server_conf='[ req ] default_bits = 2048 prompt = no default_md = sha256 distinguished_name = dn [ v3_ext ] authorityKeyIdentifier=keyid,issuer:always basicConstraints=CA:FALSE keyUsage=critical, digitalSignature, keyEncipherment extendedKeyUsage=serverAuth subjectAltName=@alt_names [ dn ] CN = server [ alt_names ] DNS.1 = localhost IP.1 = 127.0.0.1' + echo ' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' + cp /usr/local/pulsar/conf/standalone.conf /tmp/tidb_cdc_test/kv_client_stream_reconnect/pulsar_standalone.conf + pulsar_port=6650 + '[' oauth == mtls ']' + '[' oauth == oauth ']' + echo 'Waiting for oauth2 server to be ready...' Waiting for oauth2 server to be ready... + i=0 + nc -z localhost 9096 + oauth2-server + gen_oauth_config + echo ' { "client_id":"1234", "client_secret":"e0KVlA2EiBfjoN13olyZd2kv1KL", "audience":"cdc-api-uri", "issuer_url":"http://localhost:9096", "type": "client_credentials" }' + echo ' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderToken brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 brokerClientAuthenticationParameters={"privateKey":"file:///tmp/tidb_cdc_test/kv_client_stream_reconnect/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"} tokenSecretKey=data:;base64,U0poWDM2X0thcFlTeWJCdEpxMzVseFhfQnJyNExSVVJTa203UW1YSkdteThwVUZXOUVJT2NWUVBzeWt6OS1qag==' + echo ' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/ authPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 authParams={"privateKey":"/tmp/tidb_cdc_test/kv_client_stream_reconnect/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"}' ++ date + echo '[Fri Apr 4 15:32:36 CST 2025] <<<<<< START pulsar cluster in oauth mode in kv_client_stream_reconnect case >>>>>>' [Fri Apr 4 15:32:36 CST 2025] <<<<<< START pulsar cluster in oauth mode in kv_client_stream_reconnect case >>>>>> + echo 'Waiting for pulsar port to be ready...' Waiting for pulsar port to be ready... + i=0 + nc -z localhost 6650 + /usr/local/pulsar/bin/pulsar standalone --config /tmp/tidb_cdc_test/kv_client_stream_reconnect/pulsar_standalone.conf -nfw --metadata-dir /tmp/tidb_cdc_test/kv_client_stream_reconnect/pulsar-metadata --bookkeeper-dir /tmp/tidb_cdc_test/kv_client_stream_reconnect/pulsar-bookie + i=1 + '[' 1 -gt 20 ']' + sleep 2 + set +x + tso='457114112650117122 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 457114112650117122 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 4 15:32:37 CST 2025] <<<<<< START cdc server in foreign_key case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.foreign_key.29162918.out server --log-file /tmp/tidb_cdc_test/foreign_key/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/foreign_key/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ++ date + echo '[Fri Apr 4 15:32:36 CST 2025] <<<<<< START pulsar cluster in gc_safepoint case >>>>>>' [Fri Apr 4 15:32:36 CST 2025] <<<<<< START pulsar cluster in gc_safepoint case >>>>>> + workdir=/tmp/tidb_cdc_test/gc_safepoint + cluster_type=normal + cd /tmp/tidb_cdc_test/gc_safepoint + DEFAULT_PULSAR_HOME=/usr/local/pulsar + pulsar_dir=/usr/local/pulsar ++ cat + mtls_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderTls brokerClientTlsEnabled=true brokerClientTrustCertsFilePath=/tmp/tidb_cdc_test/gc_safepoint/ca.cert.pem brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls brokerClientAuthenticationParameters={"tlsCertFile":"/tmp/tidb_cdc_test/gc_safepoint/broker_client.cert.pem","tlsKeyFile":"/tmp/tidb_cdc_test/gc_safepoint/broker_client.key-pk8.pem"} brokerServicePortTls=6651 webServicePortTls=8443 tlsTrustCertsFilePath=/tmp/tidb_cdc_test/gc_safepoint/ca.cert.pem tlsCertificateFilePath=/tmp/tidb_cdc_test/gc_safepoint/server.cert.pem tlsKeyFilePath=/tmp/tidb_cdc_test/gc_safepoint/server.key-pk8.pem tlsRequireTrustedClientCertOnConnect=true tlsAllowInsecureConnection=false tlsCertRefreshCheckDurationSec=300' ++ cat + normal_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' ++ cat + mtls_client_conf=' webServiceUrl=https://localhost:8443/ brokerServiceUrl=pulsar+ssl://localhost:6651/ authPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls authParams=tlsCertFile:/tmp/tidb_cdc_test/gc_safepoint/broker_client.cert.pem,tlsKeyFile:/tmp/tidb_cdc_test/gc_safepoint/broker_client.key-pk8.pem tlsTrustCertsFilePath=/tmp/tidb_cdc_test/gc_safepoint/ca.cert.pem' ++ cat + oauth_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/ authPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 authParams={"privateKey":"/tmp/tidb_cdc_test/gc_safepoint/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"}' ++ cat + oauth_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderToken brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 brokerClientAuthenticationParameters={"privateKey":"file:///tmp/tidb_cdc_test/gc_safepoint/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"} tokenSecretKey=data:;base64,U0poWDM2X0thcFlTeWJCdEpxMzVseFhfQnJyNExSVVJTa203UW1YSkdteThwVUZXOUVJT2NWUVBzeWt6OS1qag==' ++ cat + credential_json=' { "client_id":"1234", "client_secret":"e0KVlA2EiBfjoN13olyZd2kv1KL", "audience":"cdc-api-uri", "issuer_url":"http://localhost:9096", "type": "client_credentials" }' ++ cat + cert_server_conf='[ req ] default_bits = 2048 prompt = no default_md = sha256 distinguished_name = dn [ v3_ext ] authorityKeyIdentifier=keyid,issuer:always basicConstraints=CA:FALSE keyUsage=critical, digitalSignature, keyEncipherment extendedKeyUsage=serverAuth subjectAltName=@alt_names [ dn ] CN = server [ alt_names ] DNS.1 = localhost IP.1 = 127.0.0.1' + echo ' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' + cp /usr/local/pulsar/conf/standalone.conf /tmp/tidb_cdc_test/gc_safepoint/pulsar_standalone.conf + pulsar_port=6650 + '[' normal == mtls ']' + '[' normal == oauth ']' + echo 'no cluster type specified, using default configuration.' no cluster type specified, using default configuration. ++ date + echo '[Fri Apr 4 15:32:36 CST 2025] <<<<<< START pulsar cluster in normal mode in gc_safepoint case >>>>>>' [Fri Apr 4 15:32:36 CST 2025] <<<<<< START pulsar cluster in normal mode in gc_safepoint case >>>>>> + echo 'Waiting for pulsar port to be ready...' Waiting for pulsar port to be ready... + i=0 + nc -z localhost 6650 + /usr/local/pulsar/bin/pulsar standalone --config /tmp/tidb_cdc_test/gc_safepoint/pulsar_standalone.conf -nfw --metadata-dir /tmp/tidb_cdc_test/gc_safepoint/pulsar-metadata --bookkeeper-dir /tmp/tidb_cdc_test/gc_safepoint/pulsar-bookie + i=1 + '[' 1 -gt 20 ']' + sleep 2 Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-3484/tiflow-cdc) 4030750720 bytes in 8.24 secs (489169784 bytes/sec) [Pipeline] { [Pipeline] cache + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 07:32:38 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [817 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/33c5db5f-d9be-4a2e-8a30-377c503a587d {"id":"33c5db5f-d9be-4a2e-8a30-377c503a587d","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743751955} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffb71c2ae8 33c5db5f-d9be-4a2e-8a30-377c503a587d /tidb/cdc/default/default/upstream/7489357576914434452 {"id":7489357576914434452,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/33c5db5f-d9be-4a2e-8a30-377c503a587d {"id":"33c5db5f-d9be-4a2e-8a30-377c503a587d","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743751955} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffb71c2ae8 33c5db5f-d9be-4a2e-8a30-377c503a587d /tidb/cdc/default/default/upstream/7489357576914434452 {"id":7489357576914434452,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/33c5db5f-d9be-4a2e-8a30-377c503a587d {"id":"33c5db5f-d9be-4a2e-8a30-377c503a587d","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743751955} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffb71c2ae8 33c5db5f-d9be-4a2e-8a30-377c503a587d /tidb/cdc/default/default/upstream/7489357576914434452 {"id":7489357576914434452,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x ++ date + echo '[Fri Apr 4 15:32:38 CST 2025] <<<<<< START pulsar cluster in sink_retry case >>>>>>' [Fri Apr 4 15:32:38 CST 2025] <<<<<< START pulsar cluster in sink_retry case >>>>>> + workdir=/tmp/tidb_cdc_test/sink_retry + cluster_type=normal + cd /tmp/tidb_cdc_test/sink_retry + DEFAULT_PULSAR_HOME=/usr/local/pulsar + pulsar_dir=/usr/local/pulsar ++ cat + mtls_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderTls brokerClientTlsEnabled=true brokerClientTrustCertsFilePath=/tmp/tidb_cdc_test/sink_retry/ca.cert.pem brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls brokerClientAuthenticationParameters={"tlsCertFile":"/tmp/tidb_cdc_test/sink_retry/broker_client.cert.pem","tlsKeyFile":"/tmp/tidb_cdc_test/sink_retry/broker_client.key-pk8.pem"} brokerServicePortTls=6651 webServicePortTls=8443 tlsTrustCertsFilePath=/tmp/tidb_cdc_test/sink_retry/ca.cert.pem tlsCertificateFilePath=/tmp/tidb_cdc_test/sink_retry/server.cert.pem tlsKeyFilePath=/tmp/tidb_cdc_test/sink_retry/server.key-pk8.pem tlsRequireTrustedClientCertOnConnect=true tlsAllowInsecureConnection=false tlsCertRefreshCheckDurationSec=300' ++ cat + normal_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' ++ cat + mtls_client_conf=' webServiceUrl=https://localhost:8443/ brokerServiceUrl=pulsar+ssl://localhost:6651/ authPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls authParams=tlsCertFile:/tmp/tidb_cdc_test/sink_retry/broker_client.cert.pem,tlsKeyFile:/tmp/tidb_cdc_test/sink_retry/broker_client.key-pk8.pem tlsTrustCertsFilePath=/tmp/tidb_cdc_test/sink_retry/ca.cert.pem' ++ cat + oauth_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/ authPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 authParams={"privateKey":"/tmp/tidb_cdc_test/sink_retry/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"}' ++ cat + oauth_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderToken brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 brokerClientAuthenticationParameters={"privateKey":"file:///tmp/tidb_cdc_test/sink_retry/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"} tokenSecretKey=data:;base64,U0poWDM2X0thcFlTeWJCdEpxMzVseFhfQnJyNExSVVJTa203UW1YSkdteThwVUZXOUVJT2NWUVBzeWt6OS1qag==' ++ cat + credential_json=' { "client_id":"1234", "client_secret":"e0KVlA2EiBfjoN13olyZd2kv1KL", "audience":"cdc-api-uri", "issuer_url":"http://localhost:9096", "type": "client_credentials" }' ++ cat + cert_server_conf='[ req ] default_bits = 2048 prompt = no default_md = sha256 distinguished_name = dn [ v3_ext ] authorityKeyIdentifier=keyid,issuer:always basicConstraints=CA:FALSE keyUsage=critical, digitalSignature, keyEncipherment extendedKeyUsage=serverAuth subjectAltName=@alt_names [ dn ] CN = server [ alt_names ] DNS.1 = localhost IP.1 = 127.0.0.1' + echo ' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' + cp /usr/local/pulsar/conf/standalone.conf /tmp/tidb_cdc_test/sink_retry/pulsar_standalone.conf + pulsar_port=6650 + '[' normal == mtls ']' + '[' normal == oauth ']' + echo 'no cluster type specified, using default configuration.' no cluster type specified, using default configuration. ++ date + echo '[Fri Apr 4 15:32:38 CST 2025] <<<<<< START pulsar cluster in normal mode in sink_retry case >>>>>>' [Fri Apr 4 15:32:38 CST 2025] <<<<<< START pulsar cluster in normal mode in sink_retry case >>>>>> + echo 'Waiting for pulsar port to be ready...' Waiting for pulsar port to be ready... + i=0 + nc -z localhost 6650 + /usr/local/pulsar/bin/pulsar standalone --config /tmp/tidb_cdc_test/sink_retry/pulsar_standalone.conf -nfw --metadata-dir /tmp/tidb_cdc_test/sink_retry/pulsar-metadata --bookkeeper-dir /tmp/tidb_cdc_test/sink_retry/pulsar-bookie + i=1 + '[' 1 -gt 20 ']' + sleep 2 [Fri Apr 4 15:32:38 CST 2025] <<<<<< START Pulsar consumer in canal_json_basic case >>>>>> + nc -z localhost 6650 + i=2 + '[' 2 -gt 20 ']' + sleep 2 + nc -z localhost 6650 + i=2 + '[' 2 -gt 20 ']' + sleep 2 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489357614556128340 TiDB Cluster ID. tikv_gc_leader_uuid 657fedd35ac000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-gr29-fxw7k, pid:4765, start at 2025-04-04 15:32:37.905508267 +0800 CST m=+1.461689268 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:34:37.914 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:32:37.917 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:22:37.917 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489357614556128340 TiDB Cluster ID. tikv_gc_leader_uuid 657fedd35ac000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-gr29-fxw7k, pid:4765, start at 2025-04-04 15:32:37.905508267 +0800 CST m=+1.461689268 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:34:37.914 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:32:37.917 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:22:37.917 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489357613189703836 TiDB Cluster ID. tikv_gc_leader_uuid 657fedd37200004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-gr29-fxw7k, pid:4829, start at 2025-04-04 15:32:37.964982912 +0800 CST m=+1.460472403 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:34:37.972 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:32:37.960 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:22:37.960 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/drop_many_tables/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/drop_many_tables/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/drop_many_tables/tiflash/log/proxy.log"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/drop_many_tables/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/drop_many_tables/tiflash/db/proxy"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 07:32:38 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [817 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/cbab2270-5e78-4b31-83ce-e2b3a518ee9b {"id":"cbab2270-5e78-4b31-83ce-e2b3a518ee9b","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743751956} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffb71daad1 cbab2270-5e78-4b31-83ce-e2b3a518ee9b /tidb/cdc/default/default/upstream/7489357581164055148 {"id":7489357581164055148,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/cbab2270-5e78-4b31-83ce-e2b3a518ee9b {"id":"cbab2270-5e78-4b31-83ce-e2b3a518ee9b","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743751956} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffb71daad1 cbab2270-5e78-4b31-83ce-e2b3a518ee9b /tidb/cdc/default/default/upstream/7489357581164055148 {"id":7489357581164055148,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/cbab2270-5e78-4b31-83ce-e2b3a518ee9b {"id":"cbab2270-5e78-4b31-83ce-e2b3a518ee9b","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743751956} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffb71daad1 cbab2270-5e78-4b31-83ce-e2b3a518ee9b /tidb/cdc/default/default/upstream/7489357581164055148 {"id":7489357581164055148,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x ++ date + echo '[Fri Apr 4 15:32:38 CST 2025] <<<<<< START pulsar cluster in charset_gbk case >>>>>>' [Fri Apr 4 15:32:38 CST 2025] <<<<<< START pulsar cluster in charset_gbk case >>>>>> + workdir=/tmp/tidb_cdc_test/charset_gbk + cluster_type=mtls + cd /tmp/tidb_cdc_test/charset_gbk + DEFAULT_PULSAR_HOME=/usr/local/pulsar + pulsar_dir=/usr/local/pulsar ++ cat + mtls_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderTls brokerClientTlsEnabled=true brokerClientTrustCertsFilePath=/tmp/tidb_cdc_test/charset_gbk/ca.cert.pem brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls brokerClientAuthenticationParameters={"tlsCertFile":"/tmp/tidb_cdc_test/charset_gbk/broker_client.cert.pem","tlsKeyFile":"/tmp/tidb_cdc_test/charset_gbk/broker_client.key-pk8.pem"} brokerServicePortTls=6651 webServicePortTls=8443 tlsTrustCertsFilePath=/tmp/tidb_cdc_test/charset_gbk/ca.cert.pem tlsCertificateFilePath=/tmp/tidb_cdc_test/charset_gbk/server.cert.pem tlsKeyFilePath=/tmp/tidb_cdc_test/charset_gbk/server.key-pk8.pem tlsRequireTrustedClientCertOnConnect=true tlsAllowInsecureConnection=false tlsCertRefreshCheckDurationSec=300' ++ cat + normal_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' ++ cat + mtls_client_conf=' webServiceUrl=https://localhost:8443/ brokerServiceUrl=pulsar+ssl://localhost:6651/ authPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls authParams=tlsCertFile:/tmp/tidb_cdc_test/charset_gbk/broker_client.cert.pem,tlsKeyFile:/tmp/tidb_cdc_test/charset_gbk/broker_client.key-pk8.pem tlsTrustCertsFilePath=/tmp/tidb_cdc_test/charset_gbk/ca.cert.pem' ++ cat + oauth_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/ authPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 authParams={"privateKey":"/tmp/tidb_cdc_test/charset_gbk/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"}' ++ cat + oauth_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderToken brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 brokerClientAuthenticationParameters={"privateKey":"file:///tmp/tidb_cdc_test/charset_gbk/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"} tokenSecretKey=data:;base64,U0poWDM2X0thcFlTeWJCdEpxMzVseFhfQnJyNExSVVJTa203UW1YSkdteThwVUZXOUVJT2NWUVBzeWt6OS1qag==' ++ cat + credential_json=' { "client_id":"1234", "client_secret":"e0KVlA2EiBfjoN13olyZd2kv1KL", "audience":"cdc-api-uri", "issuer_url":"http://localhost:9096", "type": "client_credentials" }' ++ cat + cert_server_conf='[ req ] default_bits = 2048 prompt = no default_md = sha256 distinguished_name = dn [ v3_ext ] authorityKeyIdentifier=keyid,issuer:always basicConstraints=CA:FALSE keyUsage=critical, digitalSignature, keyEncipherment extendedKeyUsage=serverAuth subjectAltName=@alt_names [ dn ] CN = server [ alt_names ] DNS.1 = localhost IP.1 = 127.0.0.1' + echo ' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' + cp /usr/local/pulsar/conf/standalone.conf /tmp/tidb_cdc_test/charset_gbk/pulsar_standalone.conf + pulsar_port=6650 + '[' mtls == mtls ']' + pulsar_port=6651 + gen_mtls_config + openssl genrsa -out ca.key.pem 2048 Generating RSA private key, 2048 bit long modulus (2 primes) ..+++++ ............................................................................................+++++ e is 65537 (0x010001) + openssl req -x509 -new -nodes -key ca.key.pem -subj /CN=CARoot -days 365 -out ca.cert.pem + openssl genrsa -out server.key.pem 2048 Generating RSA private key, 2048 bit long modulus (2 primes) ..............+++++ ...............................+++++ e is 65537 (0x010001) + openssl pkcs8 -topk8 -inform PEM -outform PEM -in server.key.pem -out server.key-pk8.pem -nocrypt + echo '[ req ] default_bits = 2048 prompt = no default_md = sha256 distinguished_name = dn [ v3_ext ] authorityKeyIdentifier=keyid,issuer:always basicConstraints=CA:FALSE keyUsage=critical, digitalSignature, keyEncipherment extendedKeyUsage=serverAuth subjectAltName=@alt_names [ dn ] CN = server [ alt_names ] DNS.1 = localhost IP.1 = 127.0.0.1' + openssl req -new -config server.conf -key server.key.pem -out server.csr.pem -sha256 + openssl x509 -req -in server.csr.pem -CA ca.cert.pem -CAkey ca.key.pem -CAcreateserial -out server.cert.pem -days 365 -extensions v3_ext -extfile server.conf -sha256 Signature ok subject=CN = server Getting CA Private Key + openssl genrsa -out broker_client.key.pem 2048 Generating RSA private key, 2048 bit long modulus (2 primes) .....................+++++ + nc -z localhost 6650 + i=2 + '[' 2 -gt 20 ']' + sleep 2 .........................................+++++ e is 65537 (0x010001) + openssl pkcs8 -topk8 -inform PEM -outform PEM -in broker_client.key.pem -out broker_client.key-pk8.pem -nocrypt + openssl req -new -subj /CN=broker_client -key broker_client.key.pem -out broker_client.csr.pem -sha256 + openssl x509 -req -in broker_client.csr.pem -CA ca.cert.pem -CAkey ca.key.pem -CAcreateserial -out broker_client.cert.pem -days 365 -sha256 Signature ok subject=CN = broker_client Getting CA Private Key + echo ' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderTls brokerClientTlsEnabled=true brokerClientTrustCertsFilePath=/tmp/tidb_cdc_test/charset_gbk/ca.cert.pem brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls brokerClientAuthenticationParameters={"tlsCertFile":"/tmp/tidb_cdc_test/charset_gbk/broker_client.cert.pem","tlsKeyFile":"/tmp/tidb_cdc_test/charset_gbk/broker_client.key-pk8.pem"} brokerServicePortTls=6651 webServicePortTls=8443 tlsTrustCertsFilePath=/tmp/tidb_cdc_test/charset_gbk/ca.cert.pem tlsCertificateFilePath=/tmp/tidb_cdc_test/charset_gbk/server.cert.pem tlsKeyFilePath=/tmp/tidb_cdc_test/charset_gbk/server.key-pk8.pem tlsRequireTrustedClientCertOnConnect=true tlsAllowInsecureConnection=false tlsCertRefreshCheckDurationSec=300' + echo ' webServiceUrl=https://localhost:8443/ brokerServiceUrl=pulsar+ssl://localhost:6651/ authPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls authParams=tlsCertFile:/tmp/tidb_cdc_test/charset_gbk/broker_client.cert.pem,tlsKeyFile:/tmp/tidb_cdc_test/charset_gbk/broker_client.key-pk8.pem tlsTrustCertsFilePath=/tmp/tidb_cdc_test/charset_gbk/ca.cert.pem' ++ date + echo '[Fri Apr 4 15:32:39 CST 2025] <<<<<< START pulsar cluster in mtls mode in charset_gbk case >>>>>>' [Fri Apr 4 15:32:39 CST 2025] <<<<<< START pulsar cluster in mtls mode in charset_gbk case >>>>>> + echo 'Waiting for pulsar port to be ready...' Waiting for pulsar port to be ready... + i=0 + nc -z localhost 6651 + /usr/local/pulsar/bin/pulsar standalone --config /tmp/tidb_cdc_test/charset_gbk/pulsar_standalone.conf -nfw --metadata-dir /tmp/tidb_cdc_test/charset_gbk/pulsar-metadata --bookkeeper-dir /tmp/tidb_cdc_test/charset_gbk/pulsar-bookie + i=1 + '[' 1 -gt 20 ']' + sleep 2 table test.finish_mark not exists for 1-th check, retry later + nc -z localhost 6650 + i=2 + '[' 2 -gt 20 ']' + sleep 2 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 07:32:40 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [817 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/fe1b3265-18bb-4ab4-94b0-e763c4d0856a {"id":"fe1b3265-18bb-4ab4-94b0-e763c4d0856a","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743751957} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffb71c20d6 fe1b3265-18bb-4ab4-94b0-e763c4d0856a /tidb/cdc/default/default/upstream/7489357587103125569 {"id":7489357587103125569,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/fe1b3265-18bb-4ab4-94b0-e763c4d0856a {"id":"fe1b3265-18bb-4ab4-94b0-e763c4d0856a","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743751957} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffb71c20d6 fe1b3265-18bb-4ab4-94b0-e763c4d0856a /tidb/cdc/default/default/upstream/7489357587103125569 {"id":7489357587103125569,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/fe1b3265-18bb-4ab4-94b0-e763c4d0856a {"id":"fe1b3265-18bb-4ab4-94b0-e763c4d0856a","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743751957} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffb71c20d6 fe1b3265-18bb-4ab4-94b0-e763c4d0856a /tidb/cdc/default/default/upstream/7489357587103125569 {"id":7489357587103125569,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x ++ date + echo '[Fri Apr 4 15:32:40 CST 2025] <<<<<< START pulsar cluster in foreign_key case >>>>>>' [Fri Apr 4 15:32:40 CST 2025] <<<<<< START pulsar cluster in foreign_key case >>>>>> + workdir=/tmp/tidb_cdc_test/foreign_key + cluster_type=normal + cd /tmp/tidb_cdc_test/foreign_key + DEFAULT_PULSAR_HOME=/usr/local/pulsar + pulsar_dir=/usr/local/pulsar ++ cat + mtls_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderTls brokerClientTlsEnabled=true brokerClientTrustCertsFilePath=/tmp/tidb_cdc_test/foreign_key/ca.cert.pem brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls brokerClientAuthenticationParameters={"tlsCertFile":"/tmp/tidb_cdc_test/foreign_key/broker_client.cert.pem","tlsKeyFile":"/tmp/tidb_cdc_test/foreign_key/broker_client.key-pk8.pem"} brokerServicePortTls=6651 webServicePortTls=8443 tlsTrustCertsFilePath=/tmp/tidb_cdc_test/foreign_key/ca.cert.pem tlsCertificateFilePath=/tmp/tidb_cdc_test/foreign_key/server.cert.pem tlsKeyFilePath=/tmp/tidb_cdc_test/foreign_key/server.key-pk8.pem tlsRequireTrustedClientCertOnConnect=true tlsAllowInsecureConnection=false tlsCertRefreshCheckDurationSec=300' ++ cat + normal_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' ++ cat + mtls_client_conf=' webServiceUrl=https://localhost:8443/ brokerServiceUrl=pulsar+ssl://localhost:6651/ authPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls authParams=tlsCertFile:/tmp/tidb_cdc_test/foreign_key/broker_client.cert.pem,tlsKeyFile:/tmp/tidb_cdc_test/foreign_key/broker_client.key-pk8.pem tlsTrustCertsFilePath=/tmp/tidb_cdc_test/foreign_key/ca.cert.pem' ++ cat + oauth_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/ authPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 authParams={"privateKey":"/tmp/tidb_cdc_test/foreign_key/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"}' ++ cat + oauth_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderToken brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 brokerClientAuthenticationParameters={"privateKey":"file:///tmp/tidb_cdc_test/foreign_key/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"} tokenSecretKey=data:;base64,U0poWDM2X0thcFlTeWJCdEpxMzVseFhfQnJyNExSVVJTa203UW1YSkdteThwVUZXOUVJT2NWUVBzeWt6OS1qag==' ++ cat + credential_json=' { "client_id":"1234", "client_secret":"e0KVlA2EiBfjoN13olyZd2kv1KL", "audience":"cdc-api-uri", "issuer_url":"http://localhost:9096", "type": "client_credentials" }' ++ cat + cert_server_conf='[ req ] default_bits = 2048 prompt = no default_md = sha256 distinguished_name = dn [ v3_ext ] authorityKeyIdentifier=keyid,issuer:always basicConstraints=CA:FALSE keyUsage=critical, digitalSignature, keyEncipherment extendedKeyUsage=serverAuth subjectAltName=@alt_names [ dn ] CN = server [ alt_names ] DNS.1 = localhost IP.1 = 127.0.0.1' + echo ' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' + cp /usr/local/pulsar/conf/standalone.conf /tmp/tidb_cdc_test/foreign_key/pulsar_standalone.conf + pulsar_port=6650 + '[' normal == mtls ']' + '[' normal == oauth ']' + echo 'no cluster type specified, using default configuration.' no cluster type specified, using default configuration. ++ date + echo '[Fri Apr 4 15:32:40 CST 2025] <<<<<< START pulsar cluster in normal mode in foreign_key case >>>>>>' [Fri Apr 4 15:32:40 CST 2025] <<<<<< START pulsar cluster in normal mode in foreign_key case >>>>>> + echo 'Waiting for pulsar port to be ready...' Waiting for pulsar port to be ready... + i=0 + nc -z localhost 6650 + /usr/local/pulsar/bin/pulsar standalone --config /tmp/tidb_cdc_test/foreign_key/pulsar_standalone.conf -nfw --metadata-dir /tmp/tidb_cdc_test/foreign_key/pulsar-metadata --bookkeeper-dir /tmp/tidb_cdc_test/foreign_key/pulsar-bookie + i=1 + '[' 1 -gt 20 ']' + sleep 2 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.drop_many_tables.cli.6135.out cli tso query --pd=http://127.0.0.1:2379 + nc -z localhost 6650 + echo 'Waiting for pulsar namespace to be ready...' Waiting for pulsar namespace to be ready... + i=0 + /usr/local/pulsar/bin/pulsar-admin namespaces list public + nc -z localhost 6650 + echo 'Waiting for pulsar namespace to be ready...' Waiting for pulsar namespace to be ready... + i=0 + /usr/local/pulsar/bin/pulsar-admin namespaces list public + nc -z localhost 6650 + echo 'Waiting for pulsar namespace to be ready...' Waiting for pulsar namespace to be ready... + i=0 + /usr/local/pulsar/bin/pulsar-admin namespaces list public + nc -z localhost 6651 + i=2 + '[' 2 -gt 20 ']' + sleep 2 table test.finish_mark not exists for 2-th check, retry later + nc -z localhost 6650 + echo 'Waiting for pulsar namespace to be ready...' Waiting for pulsar namespace to be ready... + i=0 + /usr/local/pulsar/bin/pulsar-admin namespaces list public + nc -z localhost 6650 + i=2 + '[' 2 -gt 20 ']' + sleep 2 + set +x + tso='457114114095054849 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 457114114095054849 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 4 15:32:42 CST 2025] <<<<<< START cdc server in drop_many_tables case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.drop_many_tables.61726174.out server --log-file /tmp/tidb_cdc_test/drop_many_tables/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/drop_many_tables/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 public/default ++ date + echo '[Fri Apr 4 15:32:42 CST 2025] <<<<<< pulsar is ready >>>>>>' [Fri Apr 4 15:32:42 CST 2025] <<<<<< pulsar is ready >>>>>> public/default public/default ++ date + echo '[Fri Apr 4 15:32:43 CST 2025] <<<<<< pulsar is ready >>>>>>' [Fri Apr 4 15:32:43 CST 2025] <<<<<< pulsar is ready >>>>>> [Fri Apr 4 15:32:43 CST 2025] <<<<<< START cdc server in kv_client_stream_reconnect case >>>>>> ++ date + echo '[Fri Apr 4 15:32:43 CST 2025] <<<<<< pulsar is ready >>>>>>' [Fri Apr 4 15:32:43 CST 2025] <<<<<< pulsar is ready >>>>>> [Fri Apr 4 15:32:43 CST 2025] <<<<<< START cdc server in gc_safepoint case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/pkg/txnutil/gc/InjectGcSafepointUpdateInterval=return(500)' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.gc_safepoint.32833285.out server --log-file /tmp/tidb_cdc_test/gc_safepoint/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/gc_safepoint/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/kv/kvClientForceReconnect=return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kv_client_stream_reconnect.32653267.out server --log-file /tmp/tidb_cdc_test/kv_client_stream_reconnect/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/kv_client_stream_reconnect/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table test.finish_mark not exists for 3-th check, retry later + nc -z localhost 6651 + echo 'Waiting for pulsar namespace to be ready...' Waiting for pulsar namespace to be ready... + i=0 + /usr/local/pulsar/bin/pulsar-admin namespaces list public [Fri Apr 4 15:32:43 CST 2025] <<<<<< START cdc server in processor_err_chan case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/processor/ProcessorAddTableError=1*return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.processor_err_chan.33783380.out server --log-file /tmp/tidb_cdc_test/processor_err_chan/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/processor_err_chan/cdc_data --cluster-id default --config /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/processor_err_chan/conf/server.toml --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 public/default + nc -z localhost 6650 + echo 'Waiting for pulsar namespace to be ready...' Waiting for pulsar namespace to be ready... + i=0 + /usr/local/pulsar/bin/pulsar-admin namespaces list public ++ date + echo '[Fri Apr 4 15:32:44 CST 2025] <<<<<< pulsar is ready >>>>>>' [Fri Apr 4 15:32:44 CST 2025] <<<<<< pulsar is ready >>>>>> + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sink_retry.cli.3377.out cli changefeed create --start-ts=457114112115081217 '--sink-uri=pulsar://127.0.0.1:6650/ticdc-sink-retry-test-4290?protocol=canal-json&enable-tidb-extension=true' table test.finish_mark not exists for 4-th check, retry later Create changefeed successfully! ID: bef83c1f-b965-4022-bee1-63d4e3fae167 Info: {"upstream_id":7489357576914434452,"namespace":"default","id":"bef83c1f-b965-4022-bee1-63d4e3fae167","sink_uri":"pulsar://127.0.0.1:6650/ticdc-sink-retry-test-4290?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2025-04-04T15:32:45.39376926+08:00","start_ts":457114112115081217,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"pulsar_config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-10-ga306f4894","resolved_ts":457114112115081217,"checkpoint_ts":457114112115081217,"checkpoint_time":"2025-04-04 15:32:33.564"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... Tenant not found Reason: Tenant not found + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 07:32:45 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [817 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ccfd6de5-bad3-464e-b2d0-31e5f4625d1a {"id":"ccfd6de5-bad3-464e-b2d0-31e5f4625d1a","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743751962} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffb739a9d5 ccfd6de5-bad3-464e-b2d0-31e5f4625d1a /tidb/cdc/default/default/upstream/7489357614556128340 {"id":7489357614556128340,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ccfd6de5-bad3-464e-b2d0-31e5f4625d1a {"id":"ccfd6de5-bad3-464e-b2d0-31e5f4625d1a","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743751962} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffb739a9d5 ccfd6de5-bad3-464e-b2d0-31e5f4625d1a /tidb/cdc/default/default/upstream/7489357614556128340 {"id":7489357614556128340,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ccfd6de5-bad3-464e-b2d0-31e5f4625d1a {"id":"ccfd6de5-bad3-464e-b2d0-31e5f4625d1a","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743751962} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffb739a9d5 ccfd6de5-bad3-464e-b2d0-31e5f4625d1a /tidb/cdc/default/default/upstream/7489357614556128340 {"id":7489357614556128340,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x ++ date + echo '[Fri Apr 4 15:32:45 CST 2025] <<<<<< START pulsar cluster in drop_many_tables case >>>>>>' [Fri Apr 4 15:32:45 CST 2025] <<<<<< START pulsar cluster in drop_many_tables case >>>>>> + workdir=/tmp/tidb_cdc_test/drop_many_tables + cluster_type=normal + cd /tmp/tidb_cdc_test/drop_many_tables + DEFAULT_PULSAR_HOME=/usr/local/pulsar + pulsar_dir=/usr/local/pulsar ++ cat + mtls_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderTls brokerClientTlsEnabled=true brokerClientTrustCertsFilePath=/tmp/tidb_cdc_test/drop_many_tables/ca.cert.pem brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls brokerClientAuthenticationParameters={"tlsCertFile":"/tmp/tidb_cdc_test/drop_many_tables/broker_client.cert.pem","tlsKeyFile":"/tmp/tidb_cdc_test/drop_many_tables/broker_client.key-pk8.pem"} brokerServicePortTls=6651 webServicePortTls=8443 tlsTrustCertsFilePath=/tmp/tidb_cdc_test/drop_many_tables/ca.cert.pem tlsCertificateFilePath=/tmp/tidb_cdc_test/drop_many_tables/server.cert.pem tlsKeyFilePath=/tmp/tidb_cdc_test/drop_many_tables/server.key-pk8.pem tlsRequireTrustedClientCertOnConnect=true tlsAllowInsecureConnection=false tlsCertRefreshCheckDurationSec=300' ++ cat + normal_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' ++ cat + mtls_client_conf=' webServiceUrl=https://localhost:8443/ brokerServiceUrl=pulsar+ssl://localhost:6651/ authPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls authParams=tlsCertFile:/tmp/tidb_cdc_test/drop_many_tables/broker_client.cert.pem,tlsKeyFile:/tmp/tidb_cdc_test/drop_many_tables/broker_client.key-pk8.pem tlsTrustCertsFilePath=/tmp/tidb_cdc_test/drop_many_tables/ca.cert.pem' ++ cat + oauth_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/ authPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 authParams={"privateKey":"/tmp/tidb_cdc_test/drop_many_tables/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"}' ++ cat + oauth_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderToken brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 brokerClientAuthenticationParameters={"privateKey":"file:///tmp/tidb_cdc_test/drop_many_tables/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"} tokenSecretKey=data:;base64,U0poWDM2X0thcFlTeWJCdEpxMzVseFhfQnJyNExSVVJTa203UW1YSkdteThwVUZXOUVJT2NWUVBzeWt6OS1qag==' ++ cat + credential_json=' { "client_id":"1234", "client_secret":"e0KVlA2EiBfjoN13olyZd2kv1KL", "audience":"cdc-api-uri", "issuer_url":"http://localhost:9096", "type": "client_credentials" }' ++ cat + cert_server_conf='[ req ] default_bits = 2048 prompt = no default_md = sha256 distinguished_name = dn [ v3_ext ] authorityKeyIdentifier=keyid,issuer:always basicConstraints=CA:FALSE keyUsage=critical, digitalSignature, keyEncipherment extendedKeyUsage=serverAuth subjectAltName=@alt_names [ dn ] CN = server [ alt_names ] DNS.1 = localhost IP.1 = 127.0.0.1' + echo ' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' + cp /usr/local/pulsar/conf/standalone.conf /tmp/tidb_cdc_test/drop_many_tables/pulsar_standalone.conf + pulsar_port=6650 + '[' normal == mtls ']' + '[' normal == oauth ']' + echo 'no cluster type specified, using default configuration.' no cluster type specified, using default configuration. ++ date + echo '[Fri Apr 4 15:32:45 CST 2025] <<<<<< START pulsar cluster in normal mode in drop_many_tables case >>>>>>' [Fri Apr 4 15:32:45 CST 2025] <<<<<< START pulsar cluster in normal mode in drop_many_tables case >>>>>> + echo 'Waiting for pulsar port to be ready...' Waiting for pulsar port to be ready... + i=0 + nc -z localhost 6650 + /usr/local/pulsar/bin/pulsar standalone --config /tmp/tidb_cdc_test/drop_many_tables/pulsar_standalone.conf -nfw --metadata-dir /tmp/tidb_cdc_test/drop_many_tables/pulsar-metadata --bookkeeper-dir /tmp/tidb_cdc_test/drop_many_tables/pulsar-bookie + i=1 + '[' 1 -gt 20 ']' + sleep 2 public/default + i=1 + '[' 1 -gt 20 ']' + sleep 2 ++ date + echo '[Fri Apr 4 15:32:46 CST 2025] <<<<<< pulsar is ready >>>>>>' [Fri Apr 4 15:32:46 CST 2025] <<<<<< pulsar is ready >>>>>> + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.foreign_key.cli.3385.out cli changefeed create --start-ts=457114112650117122 '--sink-uri=pulsar://127.0.0.1:6650/ticdc-foreign-key-test-2277?protocol=canal-json&enable-tidb-extension=true' --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/foreign_key/conf/cf.toml + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 07:32:46 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [817 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a057cda1-17ed-4e3a-83bb-338abcdea163 {"id":"a057cda1-17ed-4e3a-83bb-338abcdea163","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743751963} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffb71fc8f3 a057cda1-17ed-4e3a-83bb-338abcdea163 /tidb/cdc/default/default/upstream/7489357587702590182 {"id":7489357587702590182,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a057cda1-17ed-4e3a-83bb-338abcdea163 {"id":"a057cda1-17ed-4e3a-83bb-338abcdea163","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743751963} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffb71fc8f3 a057cda1-17ed-4e3a-83bb-338abcdea163 /tidb/cdc/default/default/upstream/7489357587702590182 {"id":7489357587702590182,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a057cda1-17ed-4e3a-83bb-338abcdea163 {"id":"a057cda1-17ed-4e3a-83bb-338abcdea163","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743751963} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffb71fc8f3 a057cda1-17ed-4e3a-83bb-338abcdea163 /tidb/cdc/default/default/upstream/7489357587702590182 {"id":7489357587702590182,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 07:32:46 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [817 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4e0b9f7b-740e-4c8c-9181-cdababfe1a29 {"id":"4e0b9f7b-740e-4c8c-9181-cdababfe1a29","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743751963} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffb7213dfa 4e0b9f7b-740e-4c8c-9181-cdababfe1a29 /tidb/cdc/default/default/upstream/7489357587574651632 {"id":7489357587574651632,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4e0b9f7b-740e-4c8c-9181-cdababfe1a29 {"id":"4e0b9f7b-740e-4c8c-9181-cdababfe1a29","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743751963} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffb7213dfa 4e0b9f7b-740e-4c8c-9181-cdababfe1a29 /tidb/cdc/default/default/upstream/7489357587574651632 {"id":7489357587574651632,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4e0b9f7b-740e-4c8c-9181-cdababfe1a29 {"id":"4e0b9f7b-740e-4c8c-9181-cdababfe1a29","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743751963} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffb7213dfa 4e0b9f7b-740e-4c8c-9181-cdababfe1a29 /tidb/cdc/default/default/upstream/7489357587574651632 {"id":7489357587574651632,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + set +x [Fri Apr 4 15:32:46 CST 2025] <<<<<< START Pulsar consumer in sink_retry case >>>>>> [Fri Apr 4 15:32:46 CST 2025] <<<<<< START Pulsar consumer in gc_safepoint case >>>>>> 0 check diff failed 1-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 07:32:46 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [817 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3609b3d6-6a0b-43a9-90f7-e1bca82bf37d {"id":"3609b3d6-6a0b-43a9-90f7-e1bca82bf37d","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743751964} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffb71efe51 3609b3d6-6a0b-43a9-90f7-e1bca82bf37d /tidb/cdc/default/default/upstream/7489357587056082243 {"id":7489357587056082243,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3609b3d6-6a0b-43a9-90f7-e1bca82bf37d {"id":"3609b3d6-6a0b-43a9-90f7-e1bca82bf37d","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743751964} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffb71efe51 3609b3d6-6a0b-43a9-90f7-e1bca82bf37d /tidb/cdc/default/default/upstream/7489357587056082243 {"id":7489357587056082243,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3609b3d6-6a0b-43a9-90f7-e1bca82bf37d {"id":"3609b3d6-6a0b-43a9-90f7-e1bca82bf37d","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743751964} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffb71efe51 3609b3d6-6a0b-43a9-90f7-e1bca82bf37d /tidb/cdc/default/default/upstream/7489357587056082243 {"id":7489357587056082243,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Fri Apr 4 15:32:46 CST 2025] <<<<<< START Pulsar consumer in kv_client_stream_reconnect case >>>>>> Create changefeed successfully! ID: 54ca390f-97e3-4a28-a0aa-333f551bec90 Info: {"upstream_id":7489357587103125569,"namespace":"default","id":"54ca390f-97e3-4a28-a0aa-333f551bec90","sink_uri":"pulsar://127.0.0.1:6650/ticdc-foreign-key-test-2277?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2025-04-04T15:32:47.262874991+08:00","start_ts":457114112650117122,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*","!foreign_key.t12"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"pulsar_config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-10-ga306f4894","resolved_ts":457114112650117122,"checkpoint_ts":457114112650117122,"checkpoint_time":"2025-04-04 15:32:35.605"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... [Fri Apr 4 15:32:47 CST 2025] <<<<<< START Pulsar consumer in processor_err_chan case >>>>>> check_changefeed_state http://127.0.0.1:2379 29c47705-00c2-42d0-8b13-4b5844c0dbb2 normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=29c47705-00c2-42d0-8b13-4b5844c0dbb2 + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c 29c47705-00c2-42d0-8b13-4b5844c0dbb2 -s + info='{ "upstream_id": 7489357587056082243, "namespace": "default", "id": "29c47705-00c2-42d0-8b13-4b5844c0dbb2", "state": "normal", "checkpoint_tso": 457114115646947331, "checkpoint_time": "2025-04-04 15:32:47.037", "error": null }' + echo '{ "upstream_id": 7489357587056082243, "namespace": "default", "id": "29c47705-00c2-42d0-8b13-4b5844c0dbb2", "state": "normal", "checkpoint_tso": 457114115646947331, "checkpoint_time": "2025-04-04 15:32:47.037", "error": null }' { "upstream_id": 7489357587056082243, "namespace": "default", "id": "29c47705-00c2-42d0-8b13-4b5844c0dbb2", "state": "normal", "checkpoint_tso": 457114115646947331, "checkpoint_time": "2025-04-04 15:32:47.037", "error": null } ++ echo '{' '"upstream_id":' 7489357587056082243, '"namespace":' '"default",' '"id":' '"29c47705-00c2-42d0-8b13-4b5844c0dbb2",' '"state":' '"normal",' '"checkpoint_tso":' 457114115646947331, '"checkpoint_time":' '"2025-04-04' '15:32:47.037",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7489357587056082243, '"namespace":' '"default",' '"id":' '"29c47705-00c2-42d0-8b13-4b5844c0dbb2",' '"state":' '"normal",' '"checkpoint_tso":' 457114115646947331, '"checkpoint_time":' '"2025-04-04' '15:32:47.037",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully + nc -z localhost 6650 + i=2 + '[' 2 -gt 20 ']' + sleep 2 Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-3484/tiflow-cdc) 4030750720 bytes in 8.26 secs (488131906 bytes/sec) [Pipeline] { table test.finish_mark not exists for 5-th check, retry later [Pipeline] cache + /usr/local/pulsar/bin/pulsar-admin namespaces list public check diff failed 1-th time, retry later + set +x [Fri Apr 4 15:32:48 CST 2025] <<<<<< START Pulsar consumer in foreign_key case >>>>>> table foreign_key.finish_mark not exists for 1-th check, retry later check diff failed 2-th time, retry later + nc -z localhost 6650 + echo 'Waiting for pulsar namespace to be ready...' Waiting for pulsar namespace to be ready... + i=0 + /usr/local/pulsar/bin/pulsar-admin namespaces list public table test.finish_mark not exists for 6-th check, retry later public/default ++ date + echo '[Fri Apr 4 15:32:50 CST 2025] <<<<<< pulsar is ready >>>>>>' [Fri Apr 4 15:32:50 CST 2025] <<<<<< pulsar is ready >>>>>> Create changefeed successfully! ID: 8f147944-7036-4c54-95b2-e0f8ccbb5425 Info: {"upstream_id":7489357581164055148,"namespace":"default","id":"8f147944-7036-4c54-95b2-e0f8ccbb5425","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-04T15:32:50.89837687+08:00","start_ts":457114112700710913,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-10-ga306f4894","resolved_ts":457114112700710913,"checkpoint_ts":457114112700710913,"checkpoint_time":"2025-04-04 15:32:35.798"} [Fri Apr 4 15:32:50 CST 2025] <<<<<< START Pulsar consumer in charset_gbk case >>>>>> check diff failed 2-th time, retry later check diff successfully check_safepoint_forward http://127.0.0.1:2379 7489357587702590182 457114116525654026 457114115450863617 table foreign_key.finish_mark not exists for 2-th check, retry later public/default table test.finish_mark not exists for 7-th check, retry later ++ date + echo '[Fri Apr 4 15:32:52 CST 2025] <<<<<< pulsar is ready >>>>>>' [Fri Apr 4 15:32:52 CST 2025] <<<<<< pulsar is ready >>>>>> + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.drop_many_tables.cli.6638.out cli changefeed create --start-ts=457114114095054849 '--sink-uri=pulsar://127.0.0.1:6650/ticdc-drop-tables-test-10962?protocol=canal-json&enable-tidb-extension=true' run task successfully check_changefeed_state http://127.0.0.1:2379 a17fdb46-eb4f-4bcc-ae49-56b05739b8bf stopped null + endpoints=http://127.0.0.1:2379 + changefeed_id=a17fdb46-eb4f-4bcc-ae49-56b05739b8bf + expected_state=stopped + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c a17fdb46-eb4f-4bcc-ae49-56b05739b8bf -s + info='{ "upstream_id": 7489357587702590182, "namespace": "default", "id": "a17fdb46-eb4f-4bcc-ae49-56b05739b8bf", "state": "stopped", "checkpoint_tso": 457114116787798023, "checkpoint_time": "2025-04-04 15:32:51.389", "error": null }' + echo '{ "upstream_id": 7489357587702590182, "namespace": "default", "id": "a17fdb46-eb4f-4bcc-ae49-56b05739b8bf", "state": "stopped", "checkpoint_tso": 457114116787798023, "checkpoint_time": "2025-04-04 15:32:51.389", "error": null }' { "upstream_id": 7489357587702590182, "namespace": "default", "id": "a17fdb46-eb4f-4bcc-ae49-56b05739b8bf", "state": "stopped", "checkpoint_tso": 457114116787798023, "checkpoint_time": "2025-04-04 15:32:51.389", "error": null } ++ echo '{' '"upstream_id":' 7489357587702590182, '"namespace":' '"default",' '"id":' '"a17fdb46-eb4f-4bcc-ae49-56b05739b8bf",' '"state":' '"stopped",' '"checkpoint_tso":' 457114116787798023, '"checkpoint_time":' '"2025-04-04' '15:32:51.389",' '"error":' null '}' ++ jq -r .state + state=stopped + [[ ! stopped == \s\t\o\p\p\e\d ]] ++ echo '{' '"upstream_id":' 7489357587702590182, '"namespace":' '"default",' '"id":' '"a17fdb46-eb4f-4bcc-ae49-56b05739b8bf",' '"state":' '"stopped",' '"checkpoint_tso":' 457114116787798023, '"checkpoint_time":' '"2025-04-04' '15:32:51.389",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully check_safepoint_equal http://127.0.0.1:2379 7489357587702590182 check diff successfully Create changefeed successfully! ID: 7dddebb3-afd0-421f-a144-ec49c42e2072 Info: {"upstream_id":7489357614556128340,"namespace":"default","id":"7dddebb3-afd0-421f-a144-ec49c42e2072","sink_uri":"pulsar://127.0.0.1:6650/ticdc-drop-tables-test-10962?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2025-04-04T15:32:52.895503239+08:00","start_ts":457114114095054849,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"pulsar_config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-10-ga306f4894","resolved_ts":457114114095054849,"checkpoint_ts":457114114095054849,"checkpoint_time":"2025-04-04 15:32:41.117"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... wait process cdc.test exit for 1-th time... table foreign_key.finish_mark not exists for 3-th check, retry later wait process cdc.test exit for 2-th time... table test.finish_mark not exists for 8-th check, retry later cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 4 15:32:54 CST 2025] <<<<<< run test case processor_err_chan success! >>>>>> + set +x [Fri Apr 4 15:32:54 CST 2025] <<<<<< START Pulsar consumer in drop_many_tables case >>>>>> table foreign_key.finish_mark not exists for 4-th check, retry later table test.finish_mark not exists for 9-th check, retry later run task successfully check_changefeed_state http://127.0.0.1:2379 a17fdb46-eb4f-4bcc-ae49-56b05739b8bf normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=a17fdb46-eb4f-4bcc-ae49-56b05739b8bf + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c a17fdb46-eb4f-4bcc-ae49-56b05739b8bf -s + info='{ "upstream_id": 7489357587702590182, "namespace": "default", "id": "a17fdb46-eb4f-4bcc-ae49-56b05739b8bf", "state": "normal", "checkpoint_tso": 457114116787798023, "checkpoint_time": "2025-04-04 15:32:51.389", "error": null }' + echo '{ "upstream_id": 7489357587702590182, "namespace": "default", "id": "a17fdb46-eb4f-4bcc-ae49-56b05739b8bf", "state": "normal", "checkpoint_tso": 457114116787798023, "checkpoint_time": "2025-04-04 15:32:51.389", "error": null }' { "upstream_id": 7489357587702590182, "namespace": "default", "id": "a17fdb46-eb4f-4bcc-ae49-56b05739b8bf", "state": "normal", "checkpoint_tso": 457114116787798023, "checkpoint_time": "2025-04-04 15:32:51.389", "error": null } ++ echo '{' '"upstream_id":' 7489357587702590182, '"namespace":' '"default",' '"id":' '"a17fdb46-eb4f-4bcc-ae49-56b05739b8bf",' '"state":' '"normal",' '"checkpoint_tso":' 457114116787798023, '"checkpoint_time":' '"2025-04-04' '15:32:51.389",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7489357587702590182, '"namespace":' '"default",' '"id":' '"a17fdb46-eb4f-4bcc-ae49-56b05739b8bf",' '"state":' '"normal",' '"checkpoint_tso":' 457114116787798023, '"checkpoint_time":' '"2025-04-04' '15:32:51.389",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully check_safepoint_forward http://127.0.0.1:2379 7489357587702590182 457114116787798022 457114116787798023 run task successfully table foreign_key.finish_mark not exists for 5-th check, retry later table charset_gbk_test0.t0 exists table charset_gbk_test0.t1 exists table charset_gbk_test1.t0 not exists for 1-th check, retry later check_changefeed_state http://127.0.0.1:2379 a17fdb46-eb4f-4bcc-ae49-56b05739b8bf stopped null + endpoints=http://127.0.0.1:2379 + changefeed_id=a17fdb46-eb4f-4bcc-ae49-56b05739b8bf + expected_state=stopped + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c a17fdb46-eb4f-4bcc-ae49-56b05739b8bf -s + info='{ "upstream_id": 7489357587702590182, "namespace": "default", "id": "a17fdb46-eb4f-4bcc-ae49-56b05739b8bf", "state": "stopped", "checkpoint_tso": 457114116787798023, "checkpoint_time": "2025-04-04 15:32:51.389", "error": null }' + echo '{ "upstream_id": 7489357587702590182, "namespace": "default", "id": "a17fdb46-eb4f-4bcc-ae49-56b05739b8bf", "state": "stopped", "checkpoint_tso": 457114116787798023, "checkpoint_time": "2025-04-04 15:32:51.389", "error": null }' { "upstream_id": 7489357587702590182, "namespace": "default", "id": "a17fdb46-eb4f-4bcc-ae49-56b05739b8bf", "state": "stopped", "checkpoint_tso": 457114116787798023, "checkpoint_time": "2025-04-04 15:32:51.389", "error": null } ++ echo '{' '"upstream_id":' 7489357587702590182, '"namespace":' '"default",' '"id":' '"a17fdb46-eb4f-4bcc-ae49-56b05739b8bf",' '"state":' '"stopped",' '"checkpoint_tso":' 457114116787798023, '"checkpoint_time":' '"2025-04-04' '15:32:51.389",' '"error":' null '}' ++ jq -r .state + state=stopped + [[ ! stopped == \s\t\o\p\p\e\d ]] ++ echo '{' '"upstream_id":' 7489357587702590182, '"namespace":' '"default",' '"id":' '"a17fdb46-eb4f-4bcc-ae49-56b05739b8bf",' '"state":' '"stopped",' '"checkpoint_tso":' 457114116787798023, '"checkpoint_time":' '"2025-04-04' '15:32:51.389",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully check_changefeed_state http://127.0.0.1:2379 7184f254-d94e-4fc1-be18-ce4a05064b52 normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=7184f254-d94e-4fc1-be18-ce4a05064b52 + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c 7184f254-d94e-4fc1-be18-ce4a05064b52 -s + info='{ "upstream_id": 7489357587702590182, "namespace": "default", "id": "7184f254-d94e-4fc1-be18-ce4a05064b52", "state": "normal", "checkpoint_tso": 457114118452150275, "checkpoint_time": "2025-04-04 15:32:57.738", "error": null }' + echo '{ "upstream_id": 7489357587702590182, "namespace": "default", "id": "7184f254-d94e-4fc1-be18-ce4a05064b52", "state": "normal", "checkpoint_tso": 457114118452150275, "checkpoint_time": "2025-04-04 15:32:57.738", "error": null }' { "upstream_id": 7489357587702590182, "namespace": "default", "id": "7184f254-d94e-4fc1-be18-ce4a05064b52", "state": "normal", "checkpoint_tso": 457114118452150275, "checkpoint_time": "2025-04-04 15:32:57.738", "error": null } ++ echo '{' '"upstream_id":' 7489357587702590182, '"namespace":' '"default",' '"id":' '"7184f254-d94e-4fc1-be18-ce4a05064b52",' '"state":' '"normal",' '"checkpoint_tso":' 457114118452150275, '"checkpoint_time":' '"2025-04-04' '15:32:57.738",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7489357587702590182, '"namespace":' '"default",' '"id":' '"7184f254-d94e-4fc1-be18-ce4a05064b52",' '"state":' '"normal",' '"checkpoint_tso":' 457114118452150275, '"checkpoint_time":' '"2025-04-04' '15:32:57.738",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully check_safepoint_equal http://127.0.0.1:2379 7489357587702590182 table test.finish_mark not exists for 10-th check, retry later Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-3484/tiflow-cdc) 4030750720 bytes in 8.42 secs (478892869 bytes/sec) [Pipeline] { [Pipeline] cache table drop_tables.c not exists for 1-th check, retry later table charset_gbk_test1.t0 not exists for 2-th check, retry later table foreign_key.finish_mark not exists for 6-th check, retry later table test.finish_mark not exists for 11-th check, retry later table drop_tables.c not exists for 2-th check, retry later table charset_gbk_test1.t0 exists table test.finish_mark not exists for 1-th check, retry later run task successfully Changefeed remove successfully. ID: a17fdb46-eb4f-4bcc-ae49-56b05739b8bf CheckpointTs: 457114116787798023 SinkURI: pulsar://127.0.0.1:6650/ticdc-gc-safepoint-19981?protocol=canal-json&enable-tidb-extension=true check_safepoint_forward http://127.0.0.1:2379 7489357587702590182 457114119160201217 457114118452150275 457114116787798023 table foreign_key.finish_mark not exists for 7-th check, retry later table test.finish_mark not exists for 12-th check, retry later run task successfully table test.finish_mark not exists for 2-th check, retry later table drop_tables.c not exists for 3-th check, retry later Changefeed remove successfully. ID: 7184f254-d94e-4fc1-be18-ce4a05064b52 CheckpointTs: 457114119684489221 SinkURI: pulsar://127.0.0.1:6650/ticdc-gc-safepoint-19981?protocol=canal-json&enable-tidb-extension=true check_safepoint_cleared http://127.0.0.1:2379 7489357587702590182 run task successfully table foreign_key.finish_mark not exists for 8-th check, retry later wait process cdc.test exit for 1-th time... table test.finish_mark not exists for 13-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/changefeed_reconstruct/run.sh using Sink-Type: pulsar... <<================= The 1 times to try to start tidb cluster... wait process cdc.test exit for 2-th time... table test.finish_mark not exists for 3-th check, retry later table drop_tables.c not exists for 4-th check, retry later cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 4 15:33:04 CST 2025] <<<<<< run test case gc_safepoint success! >>>>>> table foreign_key.finish_mark not exists for 9-th check, retry later table test.finish_mark not exists for 14-th check, retry later table drop_tables.c exists check diff successfully table test.finish_mark not exists for 4-th check, retry later wait process cdc.test exit for 1-th time... table foreign_key.finish_mark not exists for 10-th check, retry later wait process cdc.test exit for 2-th time... Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-3484/tiflow-cdc) 4030750720 bytes in 7.66 secs (526381381 bytes/sec) [Pipeline] { [Pipeline] cache cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 4 15:33:08 CST 2025] <<<<<< run test case drop_many_tables success! >>>>>> table test.finish_mark not exists for 15-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/changefeed_reconstruct Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... table test.finish_mark exists check table exists success check diff successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... table foreign_key.finish_mark not exists for 11-th check, retry later table test.finish_mark not exists for 16-th check, retry later cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 4 15:33:10 CST 2025] <<<<<< run test case charset_gbk success! >>>>>> Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table foreign_key.finish_mark not exists for 12-th check, retry later table test.finish_mark not exists for 17-th check, retry later Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table foreign_key.finish_mark exists check diff successfully table foreign_key.t12 does not exists table test.finish_mark exists check diff successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/changefeed_pause_resume/run.sh using Sink-Type: pulsar... <<================= The 1 times to try to start tidb cluster... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489357766670525368 TiDB Cluster ID. tikv_gc_leader_uuid 657fedf6c64000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-9bld-w6hq9, pid:4875, start at 2025-04-04 15:33:14.175953057 +0800 CST m=+1.462287651 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:35:14.185 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:33:14.187 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:23:14.187 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489357766670525368 TiDB Cluster ID. tikv_gc_leader_uuid 657fedf6c64000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-9bld-w6hq9, pid:4875, start at 2025-04-04 15:33:14.175953057 +0800 CST m=+1.462287651 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:35:14.185 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:33:14.187 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:23:14.187 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... table test.finish_mark exists check diff failed 1-th time, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489357765779537953 TiDB Cluster ID. tikv_gc_leader_uuid 657fedf6eac000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-9bld-w6hq9, pid:4960, start at 2025-04-04 15:33:14.298950866 +0800 CST m=+1.511916159 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:35:14.306 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:33:14.283 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:23:14.283 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/changefeed_reconstruct/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/changefeed_reconstruct/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/changefeed_reconstruct/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/changefeed_reconstruct/tiflash-proxy.toml"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/changefeed_reconstruct/tiflash/log/proxy.log"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 4 15:33:15 CST 2025] <<<<<< run test case foreign_key success! >>>>>> Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-3484/tiflow-cdc) 4030750720 bytes in 6.87 secs (586474522 bytes/sec) [Pipeline] { [Pipeline] cache check diff failed 2-th time, retry later ++ date + echo '[Fri Apr 4 15:33:17 CST 2025] <<<<<< START pulsar cluster in changefeed_reconstruct case >>>>>>' [Fri Apr 4 15:33:17 CST 2025] <<<<<< START pulsar cluster in changefeed_reconstruct case >>>>>> + workdir=/tmp/tidb_cdc_test/changefeed_reconstruct + cluster_type=normal + cd /tmp/tidb_cdc_test/changefeed_reconstruct + DEFAULT_PULSAR_HOME=/usr/local/pulsar + pulsar_dir=/usr/local/pulsar ++ cat + mtls_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderTls brokerClientTlsEnabled=true brokerClientTrustCertsFilePath=/tmp/tidb_cdc_test/changefeed_reconstruct/ca.cert.pem brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls brokerClientAuthenticationParameters={"tlsCertFile":"/tmp/tidb_cdc_test/changefeed_reconstruct/broker_client.cert.pem","tlsKeyFile":"/tmp/tidb_cdc_test/changefeed_reconstruct/broker_client.key-pk8.pem"} brokerServicePortTls=6651 webServicePortTls=8443 tlsTrustCertsFilePath=/tmp/tidb_cdc_test/changefeed_reconstruct/ca.cert.pem tlsCertificateFilePath=/tmp/tidb_cdc_test/changefeed_reconstruct/server.cert.pem tlsKeyFilePath=/tmp/tidb_cdc_test/changefeed_reconstruct/server.key-pk8.pem tlsRequireTrustedClientCertOnConnect=true tlsAllowInsecureConnection=false tlsCertRefreshCheckDurationSec=300' ++ cat + normal_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' ++ cat + mtls_client_conf=' webServiceUrl=https://localhost:8443/ brokerServiceUrl=pulsar+ssl://localhost:6651/ authPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls authParams=tlsCertFile:/tmp/tidb_cdc_test/changefeed_reconstruct/broker_client.cert.pem,tlsKeyFile:/tmp/tidb_cdc_test/changefeed_reconstruct/broker_client.key-pk8.pem tlsTrustCertsFilePath=/tmp/tidb_cdc_test/changefeed_reconstruct/ca.cert.pem' ++ cat + oauth_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/ authPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 authParams={"privateKey":"/tmp/tidb_cdc_test/changefeed_reconstruct/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"}' ++ cat + oauth_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderToken brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 brokerClientAuthenticationParameters={"privateKey":"file:///tmp/tidb_cdc_test/changefeed_reconstruct/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"} tokenSecretKey=data:;base64,U0poWDM2X0thcFlTeWJCdEpxMzVseFhfQnJyNExSVVJTa203UW1YSkdteThwVUZXOUVJT2NWUVBzeWt6OS1qag==' ++ cat + credential_json=' { "client_id":"1234", "client_secret":"e0KVlA2EiBfjoN13olyZd2kv1KL", "audience":"cdc-api-uri", "issuer_url":"http://localhost:9096", "type": "client_credentials" }' ++ cat + cert_server_conf='[ req ] default_bits = 2048 prompt = no default_md = sha256 distinguished_name = dn [ v3_ext ] authorityKeyIdentifier=keyid,issuer:always basicConstraints=CA:FALSE keyUsage=critical, digitalSignature, keyEncipherment extendedKeyUsage=serverAuth subjectAltName=@alt_names [ dn ] CN = server [ alt_names ] DNS.1 = localhost IP.1 = 127.0.0.1' + echo ' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' + cp /usr/local/pulsar/conf/standalone.conf /tmp/tidb_cdc_test/changefeed_reconstruct/pulsar_standalone.conf + pulsar_port=6650 + '[' normal == mtls ']' + '[' normal == oauth ']' + echo 'no cluster type specified, using default configuration.' no cluster type specified, using default configuration. ++ date + echo '[Fri Apr 4 15:33:17 CST 2025] <<<<<< START pulsar cluster in normal mode in changefeed_reconstruct case >>>>>>' [Fri Apr 4 15:33:17 CST 2025] <<<<<< START pulsar cluster in normal mode in changefeed_reconstruct case >>>>>> + echo 'Waiting for pulsar port to be ready...' Waiting for pulsar port to be ready... + i=0 + nc -z localhost 6650 + /usr/local/pulsar/bin/pulsar standalone --config /tmp/tidb_cdc_test/changefeed_reconstruct/pulsar_standalone.conf -nfw --metadata-dir /tmp/tidb_cdc_test/changefeed_reconstruct/pulsar-metadata --bookkeeper-dir /tmp/tidb_cdc_test/changefeed_reconstruct/pulsar-bookie + i=1 + '[' 1 -gt 20 ']' + sleep 2 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/processor_stop_delay/run.sh using Sink-Type: pulsar... <<================= The 1 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/changefeed_pause_resume Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... table sink_retry.finish_mark_1 exists check diff successfully ***************** properties ***************** "mysql.port"="4000" "operationcount"="0" "insertproportion"="0" "dotransactions"="false" "mysql.user"="root" "updateproportion"="0" "threadcount"="2" "requestdistribution"="uniform" "mysql.host"="127.0.0.1" "workload"="core" "readproportion"="0" "readallfields"="true" "recordcount"="10" "mysql.db"="sink_retry" "scanproportion"="0" ********************************************** Run finished, takes 3.866685ms INSERT - Takes(s): 0.0, Count: 10, OPS: 3712.6, Avg(us): 674, Min(us): 411, Max(us): 1532, 95th(us): 2000, 99th(us): 2000 check diff failed 1-th time, retry later check diff failed 3-th time, retry later + nc -z localhost 6650 + i=2 + '[' 2 -gt 20 ']' + sleep 2 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/ddl_manager/run.sh using Sink-Type: pulsar... <<================= The 1 times to try to start tidb cluster... check diff successfully check diff failed 4-th time, retry later wait process cdc.test exit for 1-th time... start tidb cluster in /tmp/tidb_cdc_test/processor_stop_delay Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... + nc -z localhost 6650 + echo 'Waiting for pulsar namespace to be ready...' Waiting for pulsar namespace to be ready... + i=0 + /usr/local/pulsar/bin/pulsar-admin namespaces list public wait process cdc.test exit for 2-th time... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 4 15:33:22 CST 2025] <<<<<< run test case kv_client_stream_reconnect success! >>>>>> check diff failed 5-th time, retry later Verifying downstream PD is started... public/default ++ date + echo '[Fri Apr 4 15:33:23 CST 2025] <<<<<< pulsar is ready >>>>>>' [Fri Apr 4 15:33:23 CST 2025] <<<<<< pulsar is ready >>>>>> [Fri Apr 4 15:33:23 CST 2025] <<<<<< START cdc server in changefeed_reconstruct case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_reconstruct.66816683.out server --log-file /tmp/tidb_cdc_test/changefeed_reconstruct/cdcserver1.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_reconstruct/cdc_dataserver1 --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 start tidb cluster in /tmp/tidb_cdc_test/ddl_manager Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/ddl_puller_lag/run.sh using Sink-Type: pulsar... <<================= The 1 times to try to start tidb cluster... Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-3484/tiflow-cdc) 4030750720 bytes in 6.74 secs (597837231 bytes/sec) [Pipeline] { [Pipeline] cache check diff failed 6-th time, retry later Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489357815346703245 TiDB Cluster ID. tikv_gc_leader_uuid 657fee01c2c000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-qg8g-pl0pp, pid:5267, start at 2025-04-04 15:33:25.436929895 +0800 CST m=+1.474481518 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:35:25.444 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:33:25.437 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:23:25.437 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489357815346703245 TiDB Cluster ID. tikv_gc_leader_uuid 657fee01c2c000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-qg8g-pl0pp, pid:5267, start at 2025-04-04 15:33:25.436929895 +0800 CST m=+1.474481518 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:35:25.444 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:33:25.437 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:23:25.437 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489357815969365900 TiDB Cluster ID. tikv_gc_leader_uuid 657fee01e68000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-qg8g-pl0pp, pid:5344, start at 2025-04-04 15:33:25.55074229 +0800 CST m=+1.514362268 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:35:25.558 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:33:25.530 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:23:25.530 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/changefeed_pause_resume/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/changefeed_pause_resume/tiflash/log/error.log arg matches is ArgMatches { args: {"config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/changefeed_pause_resume/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/changefeed_pause_resume/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/changefeed_pause_resume/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 07:33:26 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [817 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6945d303-ea78-4da3-960a-22400a787f1e {"id":"6945d303-ea78-4da3-960a-22400a787f1e","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752003} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffb7c17ef3 6945d303-ea78-4da3-960a-22400a787f1e /tidb/cdc/default/default/upstream/7489357766670525368 {"id":7489357766670525368,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6945d303-ea78-4da3-960a-22400a787f1e {"id":"6945d303-ea78-4da3-960a-22400a787f1e","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752003} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffb7c17ef3 6945d303-ea78-4da3-960a-22400a787f1e /tidb/cdc/default/default/upstream/7489357766670525368 {"id":7489357766670525368,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6945d303-ea78-4da3-960a-22400a787f1e {"id":"6945d303-ea78-4da3-960a-22400a787f1e","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752003} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffb7c17ef3 6945d303-ea78-4da3-960a-22400a787f1e /tidb/cdc/default/default/upstream/7489357766670525368 {"id":7489357766670525368,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Fri Apr 4 15:33:27 CST 2025] <<<<<< START Pulsar consumer in changefeed_reconstruct case >>>>>> ***************** properties ***************** "workload"="core" "scanproportion"="0" "insertproportion"="0" "requestdistribution"="uniform" "readproportion"="0" "readallfields"="true" "updateproportion"="0" "mysql.host"="127.0.0.1" "mysql.port"="4000" "recordcount"="50" "mysql.db"="changefeed_reconstruct" "dotransactions"="false" "threadcount"="4" "mysql.user"="root" "operationcount"="0" ********************************************** Run finished, takes 19.595282ms INSERT - Takes(s): 0.0, Count: 48, OPS: 3161.8, Avg(us): 1501, Min(us): 989, Max(us): 4458, 95th(us): 5000, 99th(us): 5000 check diff failed 7-th time, retry later table changefeed_reconstruct.usertable not exists for 1-th check, retry later Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ++ date + echo '[Fri Apr 4 15:33:28 CST 2025] <<<<<< START pulsar cluster in changefeed_pause_resume case >>>>>>' [Fri Apr 4 15:33:28 CST 2025] <<<<<< START pulsar cluster in changefeed_pause_resume case >>>>>> + workdir=/tmp/tidb_cdc_test/changefeed_pause_resume + cluster_type=normal + cd /tmp/tidb_cdc_test/changefeed_pause_resume + DEFAULT_PULSAR_HOME=/usr/local/pulsar + pulsar_dir=/usr/local/pulsar ++ cat + mtls_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderTls brokerClientTlsEnabled=true brokerClientTrustCertsFilePath=/tmp/tidb_cdc_test/changefeed_pause_resume/ca.cert.pem brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls brokerClientAuthenticationParameters={"tlsCertFile":"/tmp/tidb_cdc_test/changefeed_pause_resume/broker_client.cert.pem","tlsKeyFile":"/tmp/tidb_cdc_test/changefeed_pause_resume/broker_client.key-pk8.pem"} brokerServicePortTls=6651 webServicePortTls=8443 tlsTrustCertsFilePath=/tmp/tidb_cdc_test/changefeed_pause_resume/ca.cert.pem tlsCertificateFilePath=/tmp/tidb_cdc_test/changefeed_pause_resume/server.cert.pem tlsKeyFilePath=/tmp/tidb_cdc_test/changefeed_pause_resume/server.key-pk8.pem tlsRequireTrustedClientCertOnConnect=true tlsAllowInsecureConnection=false tlsCertRefreshCheckDurationSec=300' ++ cat + normal_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' ++ cat + mtls_client_conf=' webServiceUrl=https://localhost:8443/ brokerServiceUrl=pulsar+ssl://localhost:6651/ authPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls authParams=tlsCertFile:/tmp/tidb_cdc_test/changefeed_pause_resume/broker_client.cert.pem,tlsKeyFile:/tmp/tidb_cdc_test/changefeed_pause_resume/broker_client.key-pk8.pem tlsTrustCertsFilePath=/tmp/tidb_cdc_test/changefeed_pause_resume/ca.cert.pem' ++ cat + oauth_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/ authPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 authParams={"privateKey":"/tmp/tidb_cdc_test/changefeed_pause_resume/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"}' ++ cat + oauth_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderToken brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 brokerClientAuthenticationParameters={"privateKey":"file:///tmp/tidb_cdc_test/changefeed_pause_resume/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"} tokenSecretKey=data:;base64,U0poWDM2X0thcFlTeWJCdEpxMzVseFhfQnJyNExSVVJTa203UW1YSkdteThwVUZXOUVJT2NWUVBzeWt6OS1qag==' ++ cat + credential_json=' { "client_id":"1234", "client_secret":"e0KVlA2EiBfjoN13olyZd2kv1KL", "audience":"cdc-api-uri", "issuer_url":"http://localhost:9096", "type": "client_credentials" }' ++ cat + cert_server_conf='[ req ] default_bits = 2048 prompt = no default_md = sha256 distinguished_name = dn [ v3_ext ] authorityKeyIdentifier=keyid,issuer:always basicConstraints=CA:FALSE keyUsage=critical, digitalSignature, keyEncipherment extendedKeyUsage=serverAuth subjectAltName=@alt_names [ dn ] CN = server [ alt_names ] DNS.1 = localhost IP.1 = 127.0.0.1' + echo ' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' + cp /usr/local/pulsar/conf/standalone.conf /tmp/tidb_cdc_test/changefeed_pause_resume/pulsar_standalone.conf + pulsar_port=6650 + '[' normal == mtls ']' + '[' normal == oauth ']' + echo 'no cluster type specified, using default configuration.' no cluster type specified, using default configuration. ++ date + echo '[Fri Apr 4 15:33:28 CST 2025] <<<<<< START pulsar cluster in normal mode in changefeed_pause_resume case >>>>>>' [Fri Apr 4 15:33:28 CST 2025] <<<<<< START pulsar cluster in normal mode in changefeed_pause_resume case >>>>>> + echo 'Waiting for pulsar port to be ready...' Waiting for pulsar port to be ready... + i=0 + nc -z localhost 6650 + /usr/local/pulsar/bin/pulsar standalone --config /tmp/tidb_cdc_test/changefeed_pause_resume/pulsar_standalone.conf -nfw --metadata-dir /tmp/tidb_cdc_test/changefeed_pause_resume/pulsar-metadata --bookkeeper-dir /tmp/tidb_cdc_test/changefeed_pause_resume/pulsar-bookie + i=1 + '[' 1 -gt 20 ']' + sleep 2 start tidb cluster in /tmp/tidb_cdc_test/ddl_puller_lag Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489357818137836489 TiDB Cluster ID. tikv_gc_leader_uuid 657fee03b3c000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-gr29-fxw7k, pid:7832, start at 2025-04-04 15:33:27.390368944 +0800 CST m=+1.456745348 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:35:27.398 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:33:27.375 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:23:27.375 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489357818137836489 TiDB Cluster ID. tikv_gc_leader_uuid 657fee03b3c000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-gr29-fxw7k, pid:7832, start at 2025-04-04 15:33:27.390368944 +0800 CST m=+1.456745348 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:35:27.398 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:33:27.375 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:23:27.375 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489357822609105937 TiDB Cluster ID. tikv_gc_leader_uuid 657fee03bf40003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-gr29-fxw7k, pid:7896, start at 2025-04-04 15:33:27.423967045 +0800 CST m=+1.428509175 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:35:27.431 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:33:27.421 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:23:27.421 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/processor_stop_delay/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/processor_stop_delay/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/processor_stop_delay/tiflash-proxy.toml"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/processor_stop_delay/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/processor_stop_delay/tiflash/db/proxy"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... table changefeed_reconstruct.usertable exists check diff failed 8-th time, retry later check diff failed 1-th time, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489357833116153411 TiDB Cluster ID. tikv_gc_leader_uuid 657fee056f40012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-sfgg-lkh0k, pid:4717, start at 2025-04-04 15:33:29.177752122 +0800 CST m=+1.552386627 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:35:29.186 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:33:29.149 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:23:29.149 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489357833116153411 TiDB Cluster ID. tikv_gc_leader_uuid 657fee056f40012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-sfgg-lkh0k, pid:4717, start at 2025-04-04 15:33:29.177752122 +0800 CST m=+1.552386627 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:35:29.186 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:33:29.149 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:23:29.149 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489357832134625266 TiDB Cluster ID. tikv_gc_leader_uuid 657fee057b0000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-sfgg-lkh0k, pid:4782, start at 2025-04-04 15:33:29.236445477 +0800 CST m=+1.546867886 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:35:29.244 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:33:29.246 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:23:29.246 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/ddl_manager/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/ddl_manager/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/ddl_manager/tiflash/db/proxy"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/ddl_manager/tiflash-proxy.toml"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/ddl_manager/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } ++ date + echo '[Fri Apr 4 15:33:30 CST 2025] <<<<<< START pulsar cluster in processor_stop_delay case >>>>>>' [Fri Apr 4 15:33:30 CST 2025] <<<<<< START pulsar cluster in processor_stop_delay case >>>>>> + workdir=/tmp/tidb_cdc_test/processor_stop_delay + cluster_type=normal + cd /tmp/tidb_cdc_test/processor_stop_delay + DEFAULT_PULSAR_HOME=/usr/local/pulsar + pulsar_dir=/usr/local/pulsar ++ cat + mtls_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderTls brokerClientTlsEnabled=true brokerClientTrustCertsFilePath=/tmp/tidb_cdc_test/processor_stop_delay/ca.cert.pem brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls brokerClientAuthenticationParameters={"tlsCertFile":"/tmp/tidb_cdc_test/processor_stop_delay/broker_client.cert.pem","tlsKeyFile":"/tmp/tidb_cdc_test/processor_stop_delay/broker_client.key-pk8.pem"} brokerServicePortTls=6651 webServicePortTls=8443 tlsTrustCertsFilePath=/tmp/tidb_cdc_test/processor_stop_delay/ca.cert.pem tlsCertificateFilePath=/tmp/tidb_cdc_test/processor_stop_delay/server.cert.pem tlsKeyFilePath=/tmp/tidb_cdc_test/processor_stop_delay/server.key-pk8.pem tlsRequireTrustedClientCertOnConnect=true tlsAllowInsecureConnection=false tlsCertRefreshCheckDurationSec=300' ++ cat + normal_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' ++ cat + mtls_client_conf=' webServiceUrl=https://localhost:8443/ brokerServiceUrl=pulsar+ssl://localhost:6651/ authPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls authParams=tlsCertFile:/tmp/tidb_cdc_test/processor_stop_delay/broker_client.cert.pem,tlsKeyFile:/tmp/tidb_cdc_test/processor_stop_delay/broker_client.key-pk8.pem tlsTrustCertsFilePath=/tmp/tidb_cdc_test/processor_stop_delay/ca.cert.pem' ++ cat + oauth_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/ authPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 authParams={"privateKey":"/tmp/tidb_cdc_test/processor_stop_delay/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"}' ++ cat + oauth_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderToken brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 brokerClientAuthenticationParameters={"privateKey":"file:///tmp/tidb_cdc_test/processor_stop_delay/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"} tokenSecretKey=data:;base64,U0poWDM2X0thcFlTeWJCdEpxMzVseFhfQnJyNExSVVJTa203UW1YSkdteThwVUZXOUVJT2NWUVBzeWt6OS1qag==' ++ cat + credential_json=' { "client_id":"1234", "client_secret":"e0KVlA2EiBfjoN13olyZd2kv1KL", "audience":"cdc-api-uri", "issuer_url":"http://localhost:9096", "type": "client_credentials" }' ++ cat + cert_server_conf='[ req ] default_bits = 2048 prompt = no default_md = sha256 distinguished_name = dn [ v3_ext ] authorityKeyIdentifier=keyid,issuer:always basicConstraints=CA:FALSE keyUsage=critical, digitalSignature, keyEncipherment extendedKeyUsage=serverAuth subjectAltName=@alt_names [ dn ] CN = server [ alt_names ] DNS.1 = localhost IP.1 = 127.0.0.1' + echo ' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' + cp /usr/local/pulsar/conf/standalone.conf /tmp/tidb_cdc_test/processor_stop_delay/pulsar_standalone.conf + pulsar_port=6650 + '[' normal == mtls ']' + '[' normal == oauth ']' + echo 'no cluster type specified, using default configuration.' no cluster type specified, using default configuration. ++ date + echo '[Fri Apr 4 15:33:30 CST 2025] <<<<<< START pulsar cluster in normal mode in processor_stop_delay case >>>>>>' [Fri Apr 4 15:33:30 CST 2025] <<<<<< START pulsar cluster in normal mode in processor_stop_delay case >>>>>> + echo 'Waiting for pulsar port to be ready...' Waiting for pulsar port to be ready... + i=0 + nc -z localhost 6650 + /usr/local/pulsar/bin/pulsar standalone --config /tmp/tidb_cdc_test/processor_stop_delay/pulsar_standalone.conf -nfw --metadata-dir /tmp/tidb_cdc_test/processor_stop_delay/pulsar-metadata --bookkeeper-dir /tmp/tidb_cdc_test/processor_stop_delay/pulsar-bookie + i=1 + '[' 1 -gt 20 ']' + sleep 2 + nc -z localhost 6650 + i=2 + '[' 2 -gt 20 ']' + sleep 2 check diff successfully check diff failed 9-th time, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release [Fri Apr 4 15:33:32 CST 2025] <<<<<< START cdc server in ddl_manager case >>>>>> wait process 6686 exit for 1-th time... + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ExecuteDDLSlowly=return(true)' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_manager.61566158.out server --log-file /tmp/tidb_cdc_test/ddl_manager/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_manager/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/cdc/run.sh using Sink-Type: pulsar... <<================= + nc -z localhost 6650 + i=2 + '[' 2 -gt 20 ']' + sleep 2 wait process 6686 exit for 2-th time... + nc -z localhost 6650 + echo 'Waiting for pulsar namespace to be ready...' Waiting for pulsar namespace to be ready... + i=0 + /usr/local/pulsar/bin/pulsar-admin namespaces list public /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (6686) - No such process wait process 6686 exit for 3-th time... process 6686 already exit check_no_capture http://127.0.0.1:2379 parse error: Invalid numeric literal at line 1, column 6 run task successfully [Fri Apr 4 15:33:33 CST 2025] <<<<<< START cdc server in changefeed_reconstruct case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_reconstruct.69866988.out server --log-file /tmp/tidb_cdc_test/changefeed_reconstruct/cdcserver2.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_reconstruct/cdc_dataserver2 --cluster-id default --addr 127.0.0.1:8300 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff successfully Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... public/default ++ date + echo '[Fri Apr 4 15:33:34 CST 2025] <<<<<< pulsar is ready >>>>>>' [Fri Apr 4 15:33:34 CST 2025] <<<<<< pulsar is ready >>>>>> [Fri Apr 4 15:33:34 CST 2025] <<<<<< START cdc server in changefeed_pause_resume case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_pause_resume.70807082.out server --log-file /tmp/tidb_cdc_test/changefeed_pause_resume/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_pause_resume/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + nc -z localhost 6650 + echo 'Waiting for pulsar namespace to be ready...' Waiting for pulsar namespace to be ready... + i=0 + /usr/local/pulsar/bin/pulsar-admin namespaces list public cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 4 15:33:35 CST 2025] <<<<<< run test case canal_json_basic success! >>>>>> + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 07:33:35 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [817 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3bdfc680-df28-47b8-9057-168068a715f4 {"id":"3bdfc680-df28-47b8-9057-168068a715f4","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752012} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffb7ffa4cf 3bdfc680-df28-47b8-9057-168068a715f4 /tidb/cdc/default/default/upstream/7489357833116153411 {"id":7489357833116153411,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3bdfc680-df28-47b8-9057-168068a715f4 {"id":"3bdfc680-df28-47b8-9057-168068a715f4","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752012} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffb7ffa4cf 3bdfc680-df28-47b8-9057-168068a715f4 /tidb/cdc/default/default/upstream/7489357833116153411 {"id":7489357833116153411,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3bdfc680-df28-47b8-9057-168068a715f4 {"id":"3bdfc680-df28-47b8-9057-168068a715f4","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752012} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffb7ffa4cf 3bdfc680-df28-47b8-9057-168068a715f4 /tidb/cdc/default/default/upstream/7489357833116153411 {"id":7489357833116153411,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x ++ date + echo '[Fri Apr 4 15:33:35 CST 2025] <<<<<< START pulsar cluster in ddl_manager case >>>>>>' [Fri Apr 4 15:33:35 CST 2025] <<<<<< START pulsar cluster in ddl_manager case >>>>>> + workdir=/tmp/tidb_cdc_test/ddl_manager + cluster_type=normal + cd /tmp/tidb_cdc_test/ddl_manager + DEFAULT_PULSAR_HOME=/usr/local/pulsar + pulsar_dir=/usr/local/pulsar ++ cat + mtls_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderTls brokerClientTlsEnabled=true brokerClientTrustCertsFilePath=/tmp/tidb_cdc_test/ddl_manager/ca.cert.pem brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls brokerClientAuthenticationParameters={"tlsCertFile":"/tmp/tidb_cdc_test/ddl_manager/broker_client.cert.pem","tlsKeyFile":"/tmp/tidb_cdc_test/ddl_manager/broker_client.key-pk8.pem"} brokerServicePortTls=6651 webServicePortTls=8443 tlsTrustCertsFilePath=/tmp/tidb_cdc_test/ddl_manager/ca.cert.pem tlsCertificateFilePath=/tmp/tidb_cdc_test/ddl_manager/server.cert.pem tlsKeyFilePath=/tmp/tidb_cdc_test/ddl_manager/server.key-pk8.pem tlsRequireTrustedClientCertOnConnect=true tlsAllowInsecureConnection=false tlsCertRefreshCheckDurationSec=300' ++ cat + normal_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' ++ cat + mtls_client_conf=' webServiceUrl=https://localhost:8443/ brokerServiceUrl=pulsar+ssl://localhost:6651/ authPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls authParams=tlsCertFile:/tmp/tidb_cdc_test/ddl_manager/broker_client.cert.pem,tlsKeyFile:/tmp/tidb_cdc_test/ddl_manager/broker_client.key-pk8.pem tlsTrustCertsFilePath=/tmp/tidb_cdc_test/ddl_manager/ca.cert.pem' ++ cat + oauth_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/ authPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 authParams={"privateKey":"/tmp/tidb_cdc_test/ddl_manager/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"}' ++ cat + oauth_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderToken brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 brokerClientAuthenticationParameters={"privateKey":"file:///tmp/tidb_cdc_test/ddl_manager/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"} tokenSecretKey=data:;base64,U0poWDM2X0thcFlTeWJCdEpxMzVseFhfQnJyNExSVVJTa203UW1YSkdteThwVUZXOUVJT2NWUVBzeWt6OS1qag==' ++ cat + credential_json=' { "client_id":"1234", "client_secret":"e0KVlA2EiBfjoN13olyZd2kv1KL", "audience":"cdc-api-uri", "issuer_url":"http://localhost:9096", "type": "client_credentials" }' ++ cat + cert_server_conf='[ req ] default_bits = 2048 prompt = no default_md = sha256 distinguished_name = dn [ v3_ext ] authorityKeyIdentifier=keyid,issuer:always basicConstraints=CA:FALSE keyUsage=critical, digitalSignature, keyEncipherment extendedKeyUsage=serverAuth subjectAltName=@alt_names [ dn ] CN = server [ alt_names ] DNS.1 = localhost IP.1 = 127.0.0.1' + echo ' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' + cp /usr/local/pulsar/conf/standalone.conf /tmp/tidb_cdc_test/ddl_manager/pulsar_standalone.conf + pulsar_port=6650 + '[' normal == mtls ']' + '[' normal == oauth ']' + echo 'no cluster type specified, using default configuration.' no cluster type specified, using default configuration. ++ date + echo '[Fri Apr 4 15:33:35 CST 2025] <<<<<< START pulsar cluster in normal mode in ddl_manager case >>>>>>' [Fri Apr 4 15:33:35 CST 2025] <<<<<< START pulsar cluster in normal mode in ddl_manager case >>>>>> + echo 'Waiting for pulsar port to be ready...' Waiting for pulsar port to be ready... + i=0 + nc -z localhost 6650 + /usr/local/pulsar/bin/pulsar standalone --config /tmp/tidb_cdc_test/ddl_manager/pulsar_standalone.conf -nfw --metadata-dir /tmp/tidb_cdc_test/ddl_manager/pulsar-metadata --bookkeeper-dir /tmp/tidb_cdc_test/ddl_manager/pulsar-bookie + i=1 + '[' 1 -gt 20 ']' + sleep 2 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/split_region/run.sh using Sink-Type: pulsar... <<================= The 1 times to try to start tidb cluster... Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-3484/tiflow-cdc) 4030750720 bytes in 8.41 secs (479204785 bytes/sec) [Pipeline] { [Pipeline] cache VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489357859129328567 TiDB Cluster ID. tikv_gc_leader_uuid 657fee0b450000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-2bnf-6v9zm, pid:4678, start at 2025-04-04 15:33:35.172235578 +0800 CST m=+1.474222776 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:35:35.180 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:33:35.174 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:23:35.174 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489357859129328567 TiDB Cluster ID. tikv_gc_leader_uuid 657fee0b450000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-2bnf-6v9zm, pid:4678, start at 2025-04-04 15:33:35.172235578 +0800 CST m=+1.474222776 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:35:35.180 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:33:35.174 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:23:35.174 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489357852776366533 TiDB Cluster ID. tikv_gc_leader_uuid 657fee0b6a40006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-2bnf-6v9zm, pid:4762, start at 2025-04-04 15:33:35.281240825 +0800 CST m=+1.505705660 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:35:35.289 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:33:35.273 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:23:35.273 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/ddl_puller_lag/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/ddl_puller_lag/tiflash/log/error.log arg matches is ArgMatches { args: {"memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/ddl_puller_lag/tiflash-proxy.toml"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/ddl_puller_lag/tiflash/db/proxy"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/ddl_puller_lag/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 07:33:36 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3971 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/df25d989-dd2a-4dc2-9e8e-6497b32cc8a4 {UpstreamID:7489357766670525368 Namespace:default ID:df25d989-dd2a-4dc2-9e8e-6497b32cc8a4 SinkURI:pulsar://127.0.0.1:6650/ticdc-changefeed-reconstruct-28775?protocol=canal-json&enable-tidb-extension=true CreateTime:2025-04-04 15:33:27.172679078 +0800 CST StartTs:457114126014742530 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc004335830 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-10-ga306f4894 Epoch:457114126158921736} {CheckpointTs:457114127102640135 MinTableBarrierTs:457114128413360135 AdminJobType:noop} span: {tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}, resolvedTs: 457114127102640135, checkpointTs: 457114127102640135, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ec8872a5-b743-4327-8d75-040be88653c7 {"id":"ec8872a5-b743-4327-8d75-040be88653c7","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752013} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffb7c17f91 ec8872a5-b743-4327-8d75-040be88653c7 /tidb/cdc/default/default/changefeed/info/df25d989-dd2a-4dc2-9e8e-6497b32cc8a4 {"upstream-id":7489357766670525368,"namespace":"default","changefeed-id":"df25d989-dd2a-4dc2-9e8e-6497b32cc8a4","sink-uri":"pulsar://127.0.0.1:6650/ticdc-changefeed-reconstruct-28775?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2025-04-04T15:33:27.172679078+08:00","start-ts":457114126014742530,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"pulsar-config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-10-ga306f4894","epoch":457114126158921736} /tidb/cdc/default/default/changefeed/status/df25d989-dd2a-4dc2-9e8e-6497b32cc8a4 {"checkpoint-ts":457114127102640135,"min-table-barrier-ts":457114128413360135,"admin-job-type":0} /tidb/cdc/default/default/task/position/ec8872a5-b743-4327-8d75-040be88653c7/df25d989-dd2a-4dc2-9e8e-6497b32cc8a4 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489357766670525368 {"id":7489357766670525368,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/df25d989-dd2a-4dc2-9e8e-6497b32cc8a4 {UpstreamID:7489357766670525368 Namespace:default ID:df25d989-dd2a-4dc2-9e8e-6497b32cc8a4 SinkURI:pulsar://127.0.0.1:6650/ticdc-changefeed-reconstruct-28775?protocol=canal-json&enable-tidb-extension=true CreateTime:2025-04-04 15:33:27.172679078 +0800 CST StartTs:457114126014742530 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc004335830 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-10-ga306f4894 Epoch:457114126158921736} {CheckpointTs:457114127102640135 MinTableBarrierTs:457114128413360135 AdminJobType:noop} span: {tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}, resolvedTs: 457114127102640135, checkpointTs: 457114127102640135, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ec8872a5-b743-4327-8d75-040be88653c7 {"id":"ec8872a5-b743-4327-8d75-040be88653c7","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752013} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffb7c17f91 ec8872a5-b743-4327-8d75-040be88653c7 /tidb/cdc/default/default/changefeed/info/df25d989-dd2a-4dc2-9e8e-6497b32cc8a4 {"upstream-id":7489357766670525368,"namespace":"default","changefeed-id":"df25d989-dd2a-4dc2-9e8e-6497b32cc8a4","sink-uri":"pulsar://127.0.0.1:6650/ticdc-changefeed-reconstruct-28775?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2025-04-04T15:33:27.172679078+08:00","start-ts":457114126014742530,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"pulsar-config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-10-ga306f4894","epoch":457114126158921736} /tidb/cdc/default/default/changefeed/status/df25d989-dd2a-4dc2-9e8e-6497b32cc8a4 {"checkpoint-ts":457114127102640135,"min-table-barrier-ts":457114128413360135,"admin-job-type":0} /tidb/cdc/default/default/task/position/ec8872a5-b743-4327-8d75-040be88653c7/df25d989-dd2a-4dc2-9e8e-6497b32cc8a4 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489357766670525368 {"id":7489357766670525368,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/df25d989-dd2a-4dc2-9e8e-6497b32cc8a4 {UpstreamID:7489357766670525368 Namespace:default ID:df25d989-dd2a-4dc2-9e8e-6497b32cc8a4 SinkURI:pulsar://127.0.0.1:6650/ticdc-changefeed-reconstruct-28775?protocol=canal-json&enable-tidb-extension=true CreateTime:2025-04-04 15:33:27.172679078 +0800 CST StartTs:457114126014742530 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc004335830 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-10-ga306f4894 Epoch:457114126158921736} {CheckpointTs:457114127102640135 MinTableBarrierTs:457114128413360135 AdminJobType:noop} span: {tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}, resolvedTs: 457114127102640135, checkpointTs: 457114127102640135, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ec8872a5-b743-4327-8d75-040be88653c7 {"id":"ec8872a5-b743-4327-8d75-040be88653c7","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752013} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffb7c17f91 ec8872a5-b743-4327-8d75-040be88653c7 /tidb/cdc/default/default/changefeed/info/df25d989-dd2a-4dc2-9e8e-6497b32cc8a4 {"upstream-id":7489357766670525368,"namespace":"default","changefeed-id":"df25d989-dd2a-4dc2-9e8e-6497b32cc8a4","sink-uri":"pulsar://127.0.0.1:6650/ticdc-changefeed-reconstruct-28775?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2025-04-04T15:33:27.172679078+08:00","start-ts":457114126014742530,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"pulsar-config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-10-ga306f4894","epoch":457114126158921736} /tidb/cdc/default/default/changefeed/status/df25d989-dd2a-4dc2-9e8e-6497b32cc8a4 {"checkpoint-ts":457114127102640135,"min-table-barrier-ts":457114128413360135,"admin-job-type":0} /tidb/cdc/default/default/task/position/ec8872a5-b743-4327-8d75-040be88653c7/df25d989-dd2a-4dc2-9e8e-6497b32cc8a4 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489357766670525368 {"id":7489357766670525368,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x cdc.test cli capture list --pd=http://127.0.0.1:2379 2>&1 | grep id public/default ++ date + echo '[Fri Apr 4 15:33:36 CST 2025] <<<<<< pulsar is ready >>>>>>' [Fri Apr 4 15:33:36 CST 2025] <<<<<< pulsar is ready >>>>>> [Fri Apr 4 15:33:36 CST 2025] <<<<<< START cdc server in processor_stop_delay case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/processor/processorStopDelay=1*sleep(10000)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.processor_stop_delay.97419743.out server --log-file /tmp/tidb_cdc_test/processor_stop_delay/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/processor_stop_delay/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + nc -z localhost 6650 + i=2 + '[' 2 -gt 20 ']' + sleep 2 "id": "ec8872a5-b743-4327-8d75-040be88653c7", "cluster-id": "default" run task successfully + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 07:33:37 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [817 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/243f2c87-7d4a-482a-b2ac-86319e18cd82 {"id":"243f2c87-7d4a-482a-b2ac-86319e18cd82","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752015} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffb7e986f2 243f2c87-7d4a-482a-b2ac-86319e18cd82 /tidb/cdc/default/default/upstream/7489357815346703245 {"id":7489357815346703245,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/243f2c87-7d4a-482a-b2ac-86319e18cd82 {"id":"243f2c87-7d4a-482a-b2ac-86319e18cd82","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752015} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffb7e986f2 243f2c87-7d4a-482a-b2ac-86319e18cd82 /tidb/cdc/default/default/upstream/7489357815346703245 {"id":7489357815346703245,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/243f2c87-7d4a-482a-b2ac-86319e18cd82 {"id":"243f2c87-7d4a-482a-b2ac-86319e18cd82","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752015} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffb7e986f2 243f2c87-7d4a-482a-b2ac-86319e18cd82 /tidb/cdc/default/default/upstream/7489357815346703245 {"id":7489357815346703245,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_puller_lag.cli.6128.out cli tso query --pd=http://127.0.0.1:2379 [Fri Apr 4 15:33:38 CST 2025] <<<<<< START Pulsar consumer in changefeed_pause_resume case >>>>>> start tidb cluster in /tmp/tidb_cdc_test/split_region Starting Upstream PD... table changefeed_pause_resume.t1 not exists for 1-th check, retry later Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... + nc -z localhost 6650 + echo 'Waiting for pulsar namespace to be ready...' Waiting for pulsar namespace to be ready... + i=0 + /usr/local/pulsar/bin/pulsar-admin namespaces list public capture_id: ec8872a5-b743-4327-8d75-040be88653c7 check_processor_table_count http://127.0.0.1:2379 df25d989-dd2a-4dc2-9e8e-6497b32cc8a4 ec8872a5-b743-4327-8d75-040be88653c7 1 run task successfully check_processor_table_count http://127.0.0.1:2379 df25d989-dd2a-4dc2-9e8e-6497b32cc8a4 ec8872a5-b743-4327-8d75-040be88653c7 0 table count 1 does equal to expected count 0 run task failed 1-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 07:33:39 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [817 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/723d201b-fa0f-44bc-9093-0825fc7e6189 {"id":"723d201b-fa0f-44bc-9093-0825fc7e6189","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752017} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffb7f6f6f7 723d201b-fa0f-44bc-9093-0825fc7e6189 /tidb/cdc/default/default/upstream/7489357818137836489 {"id":7489357818137836489,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/723d201b-fa0f-44bc-9093-0825fc7e6189 {"id":"723d201b-fa0f-44bc-9093-0825fc7e6189","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752017} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffb7f6f6f7 723d201b-fa0f-44bc-9093-0825fc7e6189 /tidb/cdc/default/default/upstream/7489357818137836489 {"id":7489357818137836489,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/723d201b-fa0f-44bc-9093-0825fc7e6189 {"id":"723d201b-fa0f-44bc-9093-0825fc7e6189","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752017} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffb7f6f6f7 723d201b-fa0f-44bc-9093-0825fc7e6189 /tidb/cdc/default/default/upstream/7489357818137836489 {"id":7489357818137836489,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + set +x + tso='457114129117741057 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 457114129117741057 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 4 15:33:39 CST 2025] <<<<<< START cdc server in ddl_puller_lag case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/processor/processorDDLResolved=1*sleep(180000)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_puller_lag.61796181.out server --log-file /tmp/tidb_cdc_test/ddl_puller_lag/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_puller_lag/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [Fri Apr 4 15:33:40 CST 2025] <<<<<< START Pulsar consumer in processor_stop_delay case >>>>>> table changefeed_pause_resume.t1 exists table changefeed_pause_resume.t2 not exists for 1-th check, retry later public/default ++ date + echo '[Fri Apr 4 15:33:41 CST 2025] <<<<<< pulsar is ready >>>>>>' [Fri Apr 4 15:33:41 CST 2025] <<<<<< pulsar is ready >>>>>> + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_manager.cli.6630.out cli changefeed create '--sink-uri=pulsar://127.0.0.1:6650/ticdc-ddl-mamager-test-6367?protocol=canal-json&enable-tidb-extension=true' -c=ddl-manager table processor_stop_delay.t not exists for 1-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release check_processor_table_count http://127.0.0.1:2379 df25d989-dd2a-4dc2-9e8e-6497b32cc8a4 ec8872a5-b743-4327-8d75-040be88653c7 0 run task successfully ***************** properties ***************** "requestdistribution"="uniform" "scanproportion"="0" "readproportion"="0" "updateproportion"="0" "recordcount"="50" "threadcount"="4" "readallfields"="true" "operationcount"="0" "mysql.port"="4000" "mysql.host"="127.0.0.1" "mysql.user"="root" "dotransactions"="false" "mysql.db"="changefeed_reconstruct" "insertproportion"="0" "workload"="core" ********************************************** Run finished, takes 18.849402ms INSERT - Takes(s): 0.0, Count: 48, OPS: 3161.4, Avg(us): 1460, Min(us): 868, Max(us): 3765, 95th(us): 4000, 99th(us): 4000 table changefeed_reconstruct.usertable not exists for 1-th check, retry later Create changefeed successfully! ID: ddl-manager Info: {"upstream_id":7489357833116153411,"namespace":"default","id":"ddl-manager","sink_uri":"pulsar://127.0.0.1:6650/ticdc-ddl-mamager-test-6367?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2025-04-04T15:33:42.333112167+08:00","start_ts":457114129989369859,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"pulsar_config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-10-ga306f4894","resolved_ts":457114129989369859,"checkpoint_ts":457114129989369859,"checkpoint_time":"2025-04-04 15:33:41.749"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... table changefeed_pause_resume.t2 exists table changefeed_pause_resume.t3 not exists for 1-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 07:33:43 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [817 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c76ae905-971b-4389-9636-4232fb17b601 {"id":"c76ae905-971b-4389-9636-4232fb17b601","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752020} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffb81379e2 c76ae905-971b-4389-9636-4232fb17b601 /tidb/cdc/default/default/upstream/7489357859129328567 {"id":7489357859129328567,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c76ae905-971b-4389-9636-4232fb17b601 {"id":"c76ae905-971b-4389-9636-4232fb17b601","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752020} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffb81379e2 c76ae905-971b-4389-9636-4232fb17b601 /tidb/cdc/default/default/upstream/7489357859129328567 {"id":7489357859129328567,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c76ae905-971b-4389-9636-4232fb17b601 {"id":"c76ae905-971b-4389-9636-4232fb17b601","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752020} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffb81379e2 c76ae905-971b-4389-9636-4232fb17b601 /tidb/cdc/default/default/upstream/7489357859129328567 {"id":7489357859129328567,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x ++ date + echo '[Fri Apr 4 15:33:43 CST 2025] <<<<<< START pulsar cluster in ddl_puller_lag case >>>>>>' [Fri Apr 4 15:33:43 CST 2025] <<<<<< START pulsar cluster in ddl_puller_lag case >>>>>> + workdir=/tmp/tidb_cdc_test/ddl_puller_lag + cluster_type=normal + cd /tmp/tidb_cdc_test/ddl_puller_lag + DEFAULT_PULSAR_HOME=/usr/local/pulsar + pulsar_dir=/usr/local/pulsar ++ cat + mtls_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderTls brokerClientTlsEnabled=true brokerClientTrustCertsFilePath=/tmp/tidb_cdc_test/ddl_puller_lag/ca.cert.pem brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls brokerClientAuthenticationParameters={"tlsCertFile":"/tmp/tidb_cdc_test/ddl_puller_lag/broker_client.cert.pem","tlsKeyFile":"/tmp/tidb_cdc_test/ddl_puller_lag/broker_client.key-pk8.pem"} brokerServicePortTls=6651 webServicePortTls=8443 tlsTrustCertsFilePath=/tmp/tidb_cdc_test/ddl_puller_lag/ca.cert.pem tlsCertificateFilePath=/tmp/tidb_cdc_test/ddl_puller_lag/server.cert.pem tlsKeyFilePath=/tmp/tidb_cdc_test/ddl_puller_lag/server.key-pk8.pem tlsRequireTrustedClientCertOnConnect=true tlsAllowInsecureConnection=false tlsCertRefreshCheckDurationSec=300' ++ cat + normal_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' ++ cat + mtls_client_conf=' webServiceUrl=https://localhost:8443/ brokerServiceUrl=pulsar+ssl://localhost:6651/ authPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls authParams=tlsCertFile:/tmp/tidb_cdc_test/ddl_puller_lag/broker_client.cert.pem,tlsKeyFile:/tmp/tidb_cdc_test/ddl_puller_lag/broker_client.key-pk8.pem tlsTrustCertsFilePath=/tmp/tidb_cdc_test/ddl_puller_lag/ca.cert.pem' ++ cat + oauth_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/ authPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 authParams={"privateKey":"/tmp/tidb_cdc_test/ddl_puller_lag/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"}' ++ cat + oauth_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderToken brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 brokerClientAuthenticationParameters={"privateKey":"file:///tmp/tidb_cdc_test/ddl_puller_lag/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"} tokenSecretKey=data:;base64,U0poWDM2X0thcFlTeWJCdEpxMzVseFhfQnJyNExSVVJTa203UW1YSkdteThwVUZXOUVJT2NWUVBzeWt6OS1qag==' ++ cat + credential_json=' { "client_id":"1234", "client_secret":"e0KVlA2EiBfjoN13olyZd2kv1KL", "audience":"cdc-api-uri", "issuer_url":"http://localhost:9096", "type": "client_credentials" }' ++ cat + cert_server_conf='[ req ] default_bits = 2048 prompt = no default_md = sha256 distinguished_name = dn [ v3_ext ] authorityKeyIdentifier=keyid,issuer:always basicConstraints=CA:FALSE keyUsage=critical, digitalSignature, keyEncipherment extendedKeyUsage=serverAuth subjectAltName=@alt_names [ dn ] CN = server [ alt_names ] DNS.1 = localhost IP.1 = 127.0.0.1' + echo ' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' + cp /usr/local/pulsar/conf/standalone.conf /tmp/tidb_cdc_test/ddl_puller_lag/pulsar_standalone.conf + pulsar_port=6650 + '[' normal == mtls ']' + '[' normal == oauth ']' + echo 'no cluster type specified, using default configuration.' no cluster type specified, using default configuration. ++ date + echo '[Fri Apr 4 15:33:43 CST 2025] <<<<<< START pulsar cluster in normal mode in ddl_puller_lag case >>>>>>' [Fri Apr 4 15:33:43 CST 2025] <<<<<< START pulsar cluster in normal mode in ddl_puller_lag case >>>>>> + echo 'Waiting for pulsar port to be ready...' Waiting for pulsar port to be ready... + i=0 + nc -z localhost 6650 + /usr/local/pulsar/bin/pulsar standalone --config /tmp/tidb_cdc_test/ddl_puller_lag/pulsar_standalone.conf -nfw --metadata-dir /tmp/tidb_cdc_test/ddl_puller_lag/pulsar-metadata --bookkeeper-dir /tmp/tidb_cdc_test/ddl_puller_lag/pulsar-bookie + i=1 + '[' 1 -gt 20 ']' + sleep 2 + set +x [Fri Apr 4 15:33:43 CST 2025] <<<<<< START Pulsar consumer in ddl_manager case >>>>>> table processor_stop_delay.t exists check diff failed 1-th time, retry later Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table changefeed_reconstruct.usertable exists check diff failed 1-th time, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/canal_json_content_compatible/run.sh using Sink-Type: pulsar... <<================= The 1 times to try to start tidb cluster... table changefeed_pause_resume.t3 exists + nc -z localhost 6650 + i=2 + '[' 2 -gt 20 ']' + sleep 2 Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-3484/tiflow-cdc) 4030750720 bytes in 7.63 secs (528336307 bytes/sec) [Pipeline] { check diff successfully VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489357902462835193 TiDB Cluster ID. tikv_gc_leader_uuid 657fee152dc0013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-jd09-mfgkq, pid:4945, start at 2025-04-04 15:33:45.317590816 +0800 CST m=+1.709182696 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:35:45.325 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:33:45.321 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:23:45.321 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489357902462835193 TiDB Cluster ID. tikv_gc_leader_uuid 657fee152dc0013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-jd09-mfgkq, pid:4945, start at 2025-04-04 15:33:45.317590816 +0800 CST m=+1.709182696 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:35:45.325 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:33:45.321 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:23:45.321 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489357901635529870 TiDB Cluster ID. tikv_gc_leader_uuid 657fee1554c0003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-jd09-mfgkq, pid:5002, start at 2025-04-04 15:33:45.431013043 +0800 CST m=+1.751114360 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:35:45.439 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:33:45.427 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:23:45.427 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/split_region/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/split_region/tiflash/log/error.log arg matches is ArgMatches { args: {"memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/split_region/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/split_region/tiflash/log/proxy.log"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/split_region/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [Pipeline] sh check diff failed 1-th time, retry later [Pipeline] sh check diff successfully + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh pulsar G12 [Pipeline] sh Run cases: many_pk_or_uk capture_session_done_during_task ddl_attributes GIT_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/3484/display/redirect JENKINS_URL=https://do.pingcap.net/jenkins/ EXECUTOR_NUMBER=0 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_pulsar_test","buildid":"1908056220913111045","prowjobid":"b4bfd23f-f137-430c-9c71-ddcc89dd7a6d","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","base_link":"https://github.com/pingcap/tiflow/commit/4c631d5951854ac91765b729fe3849d8e8d5ed8a","pulls":[{"number":12118,"author":"dveeden","sha":"a306f48949e037965a7c1963f500f7e0abbfba51","title":"dm: upgrade go-mysql to v1.12.0","head_ref":"new_gomysql","link":"https://github.com/pingcap/tiflow/pull/12118","commit_link":"https://github.com/pingcap/tiflow/pull/12118/commits/a306f48949e037965a7c1963f500f7e0abbfba51","author_link":"https://github.com/dveeden"}]}} TZ=Asia/Shanghai BUILD_ID=1908056220913111045 POD_LABEL=pingcap_tiflow_pull_cdc_integration_pulsar_test_3484-qj98c HOSTNAME=pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-qj98-cgzmv OLDPWD=/home/jenkins RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/3484/display/redirect?page=changes JENKINS_NODE_COOKIE=fd009050-9d8d-4404-b50d-a4d67c7b54d8 GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct JAVA_HOME=/usr/lib/jvm/jre-openjdk JOB_BASE_NAME=pull_cdc_integration_pulsar_test WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test@tmp KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true GIT_URL=https://github.com/PingCAP-QE/ci.git CLASSPATH= KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 FILE_SERVER_URL=http://fileserver.pingcap.net POD_CONTAINER=golang GOPATH=/go BUILD_NUMBER=3484 USE_BAZEL_VERSION=6.5.0 KUBERNETES_PORT=tcp://10.233.0.1:443 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ HOME=/home/jenkins NODE_NAME=pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-qj98-cgzmv HUDSON_SERVER_COOKIE=83ef27fe9acccc92 JENKINS_HOME=/var/jenkins_home JOB_NAME=pingcap/tiflow/pull_cdc_integration_pulsar_test RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/3484/display/redirect?page=tests KUBERNETES_SERVICE_PORT_HTTPS=443 KUBERNETES_PORT_443_TCP_PORT=443 GIT_PREVIOUS_COMMIT=3e31ec89ba643faa364a78dbc7f062bb55f9c486 HUDSON_HOME=/var/jenkins_home JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/display/redirect PROW_JOB_ID=b4bfd23f-f137-430c-9c71-ddcc89dd7a6d TEST_GROUP=G12 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/3484/display/redirect?page=artifacts BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/3484/ TERM=xterm STAGE_NAME=Test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/ BUILD_DISPLAY_NAME=#3484 SHLVL=5 GIT_BRANCH=origin/main BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-3484 KUBERNETES_SERVICE_PORT=443 NODE_LABELS=pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-qj98-cgzmv pingcap_tiflow_pull_cdc_integration_pulsar_test_3484-qj98c PATH=/go/bin:/root/.cargo/bin:/usr/local/go/bin/:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/gradle-7.4.2/bin:/opt/apache-maven-3.8.8/bin:/usr/local/pulsar/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../../scripts/bin GIT_PREVIOUS_SUCCESSFUL_COMMIT=3e31ec89ba643faa364a78dbc7f062bb55f9c486 KUBERNETES_SERVICE_HOST=10.233.0.1 JENKINS_SERVER_COOKIE=durable-1c247575f7a5458f3a8df09baf43395940069f8be33f18a8fa953544ffab3376 _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/many_pk_or_uk/run.sh using Sink-Type: pulsar... <<================= + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh pulsar G10 [Pipeline] sh Run cases: default_value simple cdc_server_tips event_filter sql_mode GIT_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/3484/display/redirect JENKINS_URL=https://do.pingcap.net/jenkins/ EXECUTOR_NUMBER=0 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_pulsar_test","buildid":"1908056220913111045","prowjobid":"b4bfd23f-f137-430c-9c71-ddcc89dd7a6d","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","base_link":"https://github.com/pingcap/tiflow/commit/4c631d5951854ac91765b729fe3849d8e8d5ed8a","pulls":[{"number":12118,"author":"dveeden","sha":"a306f48949e037965a7c1963f500f7e0abbfba51","title":"dm: upgrade go-mysql to v1.12.0","head_ref":"new_gomysql","link":"https://github.com/pingcap/tiflow/pull/12118","commit_link":"https://github.com/pingcap/tiflow/pull/12118/commits/a306f48949e037965a7c1963f500f7e0abbfba51","author_link":"https://github.com/dveeden"}]}} TZ=Asia/Shanghai BUILD_ID=1908056220913111045 POD_LABEL=pingcap_tiflow_pull_cdc_integration_pulsar_test_3484-b84s6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-b84s-w4c6p OLDPWD=/home/jenkins RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/3484/display/redirect?page=changes JENKINS_NODE_COOKIE=97916085-6c60-4381-b7e0-7d96f9f5deb4 GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct JAVA_HOME=/usr/lib/jvm/jre-openjdk JOB_BASE_NAME=pull_cdc_integration_pulsar_test WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test@tmp KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true GIT_URL=https://github.com/PingCAP-QE/ci.git CLASSPATH= KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 FILE_SERVER_URL=http://fileserver.pingcap.net POD_CONTAINER=golang GOPATH=/go BUILD_NUMBER=3484 USE_BAZEL_VERSION=6.5.0 KUBERNETES_PORT=tcp://10.233.0.1:443 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ HOME=/home/jenkins NODE_NAME=pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-b84s-w4c6p HUDSON_SERVER_COOKIE=83ef27fe9acccc92 JENKINS_HOME=/var/jenkins_home JOB_NAME=pingcap/tiflow/pull_cdc_integration_pulsar_test RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/3484/display/redirect?page=tests KUBERNETES_SERVICE_PORT_HTTPS=443 KUBERNETES_PORT_443_TCP_PORT=443 GIT_PREVIOUS_COMMIT=3e31ec89ba643faa364a78dbc7f062bb55f9c486 HUDSON_HOME=/var/jenkins_home JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/display/redirect PROW_JOB_ID=b4bfd23f-f137-430c-9c71-ddcc89dd7a6d TEST_GROUP=G10 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/3484/display/redirect?page=artifacts BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/3484/ TERM=xterm STAGE_NAME=Test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/ BUILD_DISPLAY_NAME=#3484 SHLVL=5 GIT_BRANCH=origin/main BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-3484 KUBERNETES_SERVICE_PORT=443 NODE_LABELS=pingcap_tiflow_pull_cdc_integration_pulsar_test_3484-b84s6 pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-b84s-w4c6p PATH=/go/bin:/root/.cargo/bin:/usr/local/go/bin/:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/gradle-7.4.2/bin:/opt/apache-maven-3.8.8/bin:/usr/local/pulsar/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../../scripts/bin GIT_PREVIOUS_SUCCESSFUL_COMMIT=3e31ec89ba643faa364a78dbc7f062bb55f9c486 KUBERNETES_SERVICE_HOST=10.233.0.1 JENKINS_SERVER_COOKIE=durable-1c247575f7a5458f3a8df09baf43395940069f8be33f18a8fa953544ffab3376 _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/default_value/run.sh using Sink-Type: pulsar... <<================= find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/simple/run.sh using Sink-Type: pulsar... <<================= The 1 times to try to start tidb cluster... + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh pulsar G16 wait process cdc.test exit for 1-th time... [Pipeline] sh + nc -z localhost 6650 + echo 'Waiting for pulsar namespace to be ready...' Waiting for pulsar namespace to be ready... + i=0 + /usr/local/pulsar/bin/pulsar-admin namespaces list public Run cases: owner_resign processor_etcd_worker_delay sink_hang GIT_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/3484/display/redirect JENKINS_URL=https://do.pingcap.net/jenkins/ EXECUTOR_NUMBER=0 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_pulsar_test","buildid":"1908056220913111045","prowjobid":"b4bfd23f-f137-430c-9c71-ddcc89dd7a6d","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","base_link":"https://github.com/pingcap/tiflow/commit/4c631d5951854ac91765b729fe3849d8e8d5ed8a","pulls":[{"number":12118,"author":"dveeden","sha":"a306f48949e037965a7c1963f500f7e0abbfba51","title":"dm: upgrade go-mysql to v1.12.0","head_ref":"new_gomysql","link":"https://github.com/pingcap/tiflow/pull/12118","commit_link":"https://github.com/pingcap/tiflow/pull/12118/commits/a306f48949e037965a7c1963f500f7e0abbfba51","author_link":"https://github.com/dveeden"}]}} TZ=Asia/Shanghai BUILD_ID=1908056220913111045 POD_LABEL=pingcap_tiflow_pull_cdc_integration_pulsar_test_3484-7fxbb HOSTNAME=pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-7fxb-ntnhx OLDPWD=/home/jenkins RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/3484/display/redirect?page=changes JENKINS_NODE_COOKIE=923790a6-c27d-4229-9628-fea9403bf9c4 GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct JAVA_HOME=/usr/lib/jvm/jre-openjdk JOB_BASE_NAME=pull_cdc_integration_pulsar_test WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test@tmp KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true GIT_URL=https://github.com/PingCAP-QE/ci.git CLASSPATH= KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 FILE_SERVER_URL=http://fileserver.pingcap.net POD_CONTAINER=golang GOPATH=/go BUILD_NUMBER=3484 USE_BAZEL_VERSION=6.5.0 KUBERNETES_PORT=tcp://10.233.0.1:443 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ HOME=/home/jenkins NODE_NAME=pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-7fxb-ntnhx HUDSON_SERVER_COOKIE=83ef27fe9acccc92 JENKINS_HOME=/var/jenkins_home JOB_NAME=pingcap/tiflow/pull_cdc_integration_pulsar_test RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/3484/display/redirect?page=tests KUBERNETES_SERVICE_PORT_HTTPS=443 KUBERNETES_PORT_443_TCP_PORT=443 GIT_PREVIOUS_COMMIT=3e31ec89ba643faa364a78dbc7f062bb55f9c486 HUDSON_HOME=/var/jenkins_home JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/display/redirect PROW_JOB_ID=b4bfd23f-f137-430c-9c71-ddcc89dd7a6d TEST_GROUP=G16 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/3484/display/redirect?page=artifacts BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/3484/ TERM=xterm STAGE_NAME=Test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/ BUILD_DISPLAY_NAME=#3484 SHLVL=5 GIT_BRANCH=origin/main BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-3484 KUBERNETES_SERVICE_PORT=443 NODE_LABELS=pingcap_tiflow_pull_cdc_integration_pulsar_test_3484-7fxbb pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-7fxb-ntnhx PATH=/go/bin:/root/.cargo/bin:/usr/local/go/bin/:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/gradle-7.4.2/bin:/opt/apache-maven-3.8.8/bin:/usr/local/pulsar/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../../scripts/bin GIT_PREVIOUS_SUCCESSFUL_COMMIT=3e31ec89ba643faa364a78dbc7f062bb55f9c486 KUBERNETES_SERVICE_HOST=10.233.0.1 JENKINS_SERVER_COOKIE=durable-1c247575f7a5458f3a8df09baf43395940069f8be33f18a8fa953544ffab3376 _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/owner_resign/run.sh using Sink-Type: pulsar... <<================= The 1 times to try to start tidb cluster... + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh pulsar G13 [Pipeline] sh Run cases: tiflash region_merge common_1 GIT_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/3484/display/redirect JENKINS_URL=https://do.pingcap.net/jenkins/ EXECUTOR_NUMBER=0 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_pulsar_test","buildid":"1908056220913111045","prowjobid":"b4bfd23f-f137-430c-9c71-ddcc89dd7a6d","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","base_link":"https://github.com/pingcap/tiflow/commit/4c631d5951854ac91765b729fe3849d8e8d5ed8a","pulls":[{"number":12118,"author":"dveeden","sha":"a306f48949e037965a7c1963f500f7e0abbfba51","title":"dm: upgrade go-mysql to v1.12.0","head_ref":"new_gomysql","link":"https://github.com/pingcap/tiflow/pull/12118","commit_link":"https://github.com/pingcap/tiflow/pull/12118/commits/a306f48949e037965a7c1963f500f7e0abbfba51","author_link":"https://github.com/dveeden"}]}} TZ=Asia/Shanghai BUILD_ID=1908056220913111045 POD_LABEL=pingcap_tiflow_pull_cdc_integration_pulsar_test_3484-vgqpb HOSTNAME=pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-vgqp-h2d14 OLDPWD=/home/jenkins RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/3484/display/redirect?page=changes JENKINS_NODE_COOKIE=4bc782be-f201-4c3c-9e7f-4eb41c2b3576 GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct JAVA_HOME=/usr/lib/jvm/jre-openjdk JOB_BASE_NAME=pull_cdc_integration_pulsar_test WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test@tmp KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true GIT_URL=https://github.com/PingCAP-QE/ci.git CLASSPATH= KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 FILE_SERVER_URL=http://fileserver.pingcap.net POD_CONTAINER=golang GOPATH=/go BUILD_NUMBER=3484 USE_BAZEL_VERSION=6.5.0 KUBERNETES_PORT=tcp://10.233.0.1:443 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ HOME=/home/jenkins NODE_NAME=pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-vgqp-h2d14 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 JENKINS_HOME=/var/jenkins_home JOB_NAME=pingcap/tiflow/pull_cdc_integration_pulsar_test RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/3484/display/redirect?page=tests KUBERNETES_SERVICE_PORT_HTTPS=443 KUBERNETES_PORT_443_TCP_PORT=443 GIT_PREVIOUS_COMMIT=3e31ec89ba643faa364a78dbc7f062bb55f9c486 HUDSON_HOME=/var/jenkins_home JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/display/redirect PROW_JOB_ID=b4bfd23f-f137-430c-9c71-ddcc89dd7a6d TEST_GROUP=G13 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/3484/display/redirect?page=artifacts BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/3484/ TERM=xterm STAGE_NAME=Test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/ BUILD_DISPLAY_NAME=#3484 SHLVL=5 GIT_BRANCH=origin/main BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-3484 KUBERNETES_SERVICE_PORT=443 NODE_LABELS=pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-vgqp-h2d14 pingcap_tiflow_pull_cdc_integration_pulsar_test_3484-vgqpb PATH=/go/bin:/root/.cargo/bin:/usr/local/go/bin/:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/gradle-7.4.2/bin:/opt/apache-maven-3.8.8/bin:/usr/local/pulsar/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../../scripts/bin GIT_PREVIOUS_SUCCESSFUL_COMMIT=3e31ec89ba643faa364a78dbc7f062bb55f9c486 KUBERNETES_SERVICE_HOST=10.233.0.1 JENKINS_SERVER_COOKIE=durable-1c247575f7a5458f3a8df09baf43395940069f8be33f18a8fa953544ffab3376 _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/tiflash/run.sh using Sink-Type: pulsar... <<================= The 1 times to try to start tidb cluster... wait process cdc.test exit for 2-th time... + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh pulsar G15 [Pipeline] sh check diff failed 1-th time, retry later Run cases: new_ci_collation batch_add_table multi_rocks ci_collation_compatibility GIT_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/3484/display/redirect JENKINS_URL=https://do.pingcap.net/jenkins/ EXECUTOR_NUMBER=0 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_pulsar_test","buildid":"1908056220913111045","prowjobid":"b4bfd23f-f137-430c-9c71-ddcc89dd7a6d","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","base_link":"https://github.com/pingcap/tiflow/commit/4c631d5951854ac91765b729fe3849d8e8d5ed8a","pulls":[{"number":12118,"author":"dveeden","sha":"a306f48949e037965a7c1963f500f7e0abbfba51","title":"dm: upgrade go-mysql to v1.12.0","head_ref":"new_gomysql","link":"https://github.com/pingcap/tiflow/pull/12118","commit_link":"https://github.com/pingcap/tiflow/pull/12118/commits/a306f48949e037965a7c1963f500f7e0abbfba51","author_link":"https://github.com/dveeden"}]}} TZ=Asia/Shanghai BUILD_ID=1908056220913111045 POD_LABEL=pingcap_tiflow_pull_cdc_integration_pulsar_test_3484-j3lwh HOSTNAME=pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-j3lw-c6tm2 OLDPWD=/home/jenkins RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/3484/display/redirect?page=changes JENKINS_NODE_COOKIE=ac83b227-dab8-46eb-ba7e-ac1e95425df9 GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct JAVA_HOME=/usr/lib/jvm/jre-openjdk JOB_BASE_NAME=pull_cdc_integration_pulsar_test WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test@tmp KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true GIT_URL=https://github.com/PingCAP-QE/ci.git CLASSPATH= KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 FILE_SERVER_URL=http://fileserver.pingcap.net POD_CONTAINER=golang GOPATH=/go BUILD_NUMBER=3484 USE_BAZEL_VERSION=6.5.0 KUBERNETES_PORT=tcp://10.233.0.1:443 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ HOME=/home/jenkins NODE_NAME=pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-j3lw-c6tm2 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 JENKINS_HOME=/var/jenkins_home JOB_NAME=pingcap/tiflow/pull_cdc_integration_pulsar_test RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/3484/display/redirect?page=tests KUBERNETES_SERVICE_PORT_HTTPS=443 KUBERNETES_PORT_443_TCP_PORT=443 GIT_PREVIOUS_COMMIT=3e31ec89ba643faa364a78dbc7f062bb55f9c486 HUDSON_HOME=/var/jenkins_home JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/display/redirect PROW_JOB_ID=b4bfd23f-f137-430c-9c71-ddcc89dd7a6d TEST_GROUP=G15 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/3484/display/redirect?page=artifacts BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/3484/ TERM=xterm STAGE_NAME=Test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/ BUILD_DISPLAY_NAME=#3484 SHLVL=5 GIT_BRANCH=origin/main BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-3484 KUBERNETES_SERVICE_PORT=443 NODE_LABELS=pingcap_tiflow_pull_cdc_integration_pulsar_test_3484-j3lwh pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-j3lw-c6tm2 PATH=/go/bin:/root/.cargo/bin:/usr/local/go/bin/:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/gradle-7.4.2/bin:/opt/apache-maven-3.8.8/bin:/usr/local/pulsar/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../../scripts/bin GIT_PREVIOUS_SUCCESSFUL_COMMIT=3e31ec89ba643faa364a78dbc7f062bb55f9c486 KUBERNETES_SERVICE_HOST=10.233.0.1 JENKINS_SERVER_COOKIE=durable-1c247575f7a5458f3a8df09baf43395940069f8be33f18a8fa953544ffab3376 _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/new_ci_collation/run.sh using Sink-Type: pulsar... <<================= + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh pulsar G17 [Pipeline] sh Run cases: clustered_index processor_resolved_ts_fallback GIT_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/3484/display/redirect JENKINS_URL=https://do.pingcap.net/jenkins/ EXECUTOR_NUMBER=0 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_pulsar_test","buildid":"1908056220913111045","prowjobid":"b4bfd23f-f137-430c-9c71-ddcc89dd7a6d","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","base_link":"https://github.com/pingcap/tiflow/commit/4c631d5951854ac91765b729fe3849d8e8d5ed8a","pulls":[{"number":12118,"author":"dveeden","sha":"a306f48949e037965a7c1963f500f7e0abbfba51","title":"dm: upgrade go-mysql to v1.12.0","head_ref":"new_gomysql","link":"https://github.com/pingcap/tiflow/pull/12118","commit_link":"https://github.com/pingcap/tiflow/pull/12118/commits/a306f48949e037965a7c1963f500f7e0abbfba51","author_link":"https://github.com/dveeden"}]}} TZ=Asia/Shanghai BUILD_ID=1908056220913111045 POD_LABEL=pingcap_tiflow_pull_cdc_integration_pulsar_test_3484-5mlns HOSTNAME=pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-5mln-gcmj2 OLDPWD=/home/jenkins RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/3484/display/redirect?page=changes JENKINS_NODE_COOKIE=26690751-6f37-4875-ae0f-750b28af0074 GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct JAVA_HOME=/usr/lib/jvm/jre-openjdk JOB_BASE_NAME=pull_cdc_integration_pulsar_test WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test@tmp KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true GIT_URL=https://github.com/PingCAP-QE/ci.git CLASSPATH= KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 FILE_SERVER_URL=http://fileserver.pingcap.net POD_CONTAINER=golang GOPATH=/go BUILD_NUMBER=3484 USE_BAZEL_VERSION=6.5.0 KUBERNETES_PORT=tcp://10.233.0.1:443 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ HOME=/home/jenkins NODE_NAME=pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-5mln-gcmj2 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 JENKINS_HOME=/var/jenkins_home JOB_NAME=pingcap/tiflow/pull_cdc_integration_pulsar_test RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/3484/display/redirect?page=tests KUBERNETES_SERVICE_PORT_HTTPS=443 KUBERNETES_PORT_443_TCP_PORT=443 GIT_PREVIOUS_COMMIT=3e31ec89ba643faa364a78dbc7f062bb55f9c486 HUDSON_HOME=/var/jenkins_home JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/display/redirect PROW_JOB_ID=b4bfd23f-f137-430c-9c71-ddcc89dd7a6d TEST_GROUP=G17 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/3484/display/redirect?page=artifacts BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/3484/ TERM=xterm STAGE_NAME=Test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/ BUILD_DISPLAY_NAME=#3484 SHLVL=5 GIT_BRANCH=origin/main BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-3484 KUBERNETES_SERVICE_PORT=443 NODE_LABELS=pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-5mln-gcmj2 pingcap_tiflow_pull_cdc_integration_pulsar_test_3484-5mlns PATH=/go/bin:/root/.cargo/bin:/usr/local/go/bin/:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/gradle-7.4.2/bin:/opt/apache-maven-3.8.8/bin:/usr/local/pulsar/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../../scripts/bin GIT_PREVIOUS_SUCCESSFUL_COMMIT=3e31ec89ba643faa364a78dbc7f062bb55f9c486 KUBERNETES_SERVICE_HOST=10.233.0.1 JENKINS_SERVER_COOKIE=durable-1c247575f7a5458f3a8df09baf43395940069f8be33f18a8fa953544ffab3376 _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/clustered_index/run.sh using Sink-Type: pulsar... <<================= [Fri Apr 4 15:33:48 CST 2025] <<<<<< skip test case clustered_index for pulsar! >>>>>> find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/processor_resolved_ts_fallback/run.sh using Sink-Type: pulsar... <<================= The 1 times to try to start tidb cluster... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 4 15:33:47 CST 2025] <<<<<< run test case changefeed_reconstruct success! >>>>>> + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh pulsar G11 [Pipeline] // timeout [Pipeline] } start tidb cluster in /tmp/tidb_cdc_test/canal_json_content_compatible Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Run cases: resolve_lock move_table autorandom generate_column GIT_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/3484/display/redirect JENKINS_URL=https://do.pingcap.net/jenkins/ EXECUTOR_NUMBER=0 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_pulsar_test","buildid":"1908056220913111045","prowjobid":"b4bfd23f-f137-430c-9c71-ddcc89dd7a6d","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","base_link":"https://github.com/pingcap/tiflow/commit/4c631d5951854ac91765b729fe3849d8e8d5ed8a","pulls":[{"number":12118,"author":"dveeden","sha":"a306f48949e037965a7c1963f500f7e0abbfba51","title":"dm: upgrade go-mysql to v1.12.0","head_ref":"new_gomysql","link":"https://github.com/pingcap/tiflow/pull/12118","commit_link":"https://github.com/pingcap/tiflow/pull/12118/commits/a306f48949e037965a7c1963f500f7e0abbfba51","author_link":"https://github.com/dveeden"}]}} TZ=Asia/Shanghai BUILD_ID=1908056220913111045 POD_LABEL=pingcap_tiflow_pull_cdc_integration_pulsar_test_3484-6nlr7 HOSTNAME=pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-6nlr-p88d4 OLDPWD=/home/jenkins RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/3484/display/redirect?page=changes JENKINS_NODE_COOKIE=f6b21f74-698d-4bc7-87de-6a147714c544 GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct JAVA_HOME=/usr/lib/jvm/jre-openjdk JOB_BASE_NAME=pull_cdc_integration_pulsar_test WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test@tmp KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true GIT_URL=https://github.com/PingCAP-QE/ci.git CLASSPATH= KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 FILE_SERVER_URL=http://fileserver.pingcap.net POD_CONTAINER=golang GOPATH=/go BUILD_NUMBER=3484 USE_BAZEL_VERSION=6.5.0 KUBERNETES_PORT=tcp://10.233.0.1:443 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ HOME=/home/jenkins NODE_NAME=pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-6nlr-p88d4 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 JENKINS_HOME=/var/jenkins_home JOB_NAME=pingcap/tiflow/pull_cdc_integration_pulsar_test RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/3484/display/redirect?page=tests KUBERNETES_SERVICE_PORT_HTTPS=443 KUBERNETES_PORT_443_TCP_PORT=443 GIT_PREVIOUS_COMMIT=3e31ec89ba643faa364a78dbc7f062bb55f9c486 HUDSON_HOME=/var/jenkins_home JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/display/redirect PROW_JOB_ID=b4bfd23f-f137-430c-9c71-ddcc89dd7a6d TEST_GROUP=G11 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/3484/display/redirect?page=artifacts BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/3484/ TERM=xterm STAGE_NAME=Test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/ BUILD_DISPLAY_NAME=#3484 SHLVL=5 GIT_BRANCH=origin/main BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-3484 KUBERNETES_SERVICE_PORT=443 NODE_LABELS=pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-6nlr-p88d4 pingcap_tiflow_pull_cdc_integration_pulsar_test_3484-6nlr7 PATH=/go/bin:/root/.cargo/bin:/usr/local/go/bin/:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/gradle-7.4.2/bin:/opt/apache-maven-3.8.8/bin:/usr/local/pulsar/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../../scripts/bin GIT_PREVIOUS_SUCCESSFUL_COMMIT=3e31ec89ba643faa364a78dbc7f062bb55f9c486 KUBERNETES_SERVICE_HOST=10.233.0.1 JENKINS_SERVER_COOKIE=durable-1c247575f7a5458f3a8df09baf43395940069f8be33f18a8fa953544ffab3376 _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/resolve_lock/run.sh using Sink-Type: pulsar... <<================= + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.split_region.cli.6307.out cli tso query --pd=http://127.0.0.1:2379 [Pipeline] // timeout [Pipeline] } + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh pulsar G14 check diff failed 2-th time, retry later Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... Run cases: changefeed_finish force_replicate_table GIT_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/3484/display/redirect JENKINS_URL=https://do.pingcap.net/jenkins/ EXECUTOR_NUMBER=0 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_pulsar_test","buildid":"1908056220913111045","prowjobid":"b4bfd23f-f137-430c-9c71-ddcc89dd7a6d","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","base_link":"https://github.com/pingcap/tiflow/commit/4c631d5951854ac91765b729fe3849d8e8d5ed8a","pulls":[{"number":12118,"author":"dveeden","sha":"a306f48949e037965a7c1963f500f7e0abbfba51","title":"dm: upgrade go-mysql to v1.12.0","head_ref":"new_gomysql","link":"https://github.com/pingcap/tiflow/pull/12118","commit_link":"https://github.com/pingcap/tiflow/pull/12118/commits/a306f48949e037965a7c1963f500f7e0abbfba51","author_link":"https://github.com/dveeden"}]}} TZ=Asia/Shanghai BUILD_ID=1908056220913111045 POD_LABEL=pingcap_tiflow_pull_cdc_integration_pulsar_test_3484-2xgwq HOSTNAME=pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-2xgw-gnw10 OLDPWD=/home/jenkins RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/3484/display/redirect?page=changes JENKINS_NODE_COOKIE=52570f5b-f7ed-4691-9c1c-9e16c5c22b2c GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct JAVA_HOME=/usr/lib/jvm/jre-openjdk JOB_BASE_NAME=pull_cdc_integration_pulsar_test WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test@tmp KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true GIT_URL=https://github.com/PingCAP-QE/ci.git CLASSPATH= KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 FILE_SERVER_URL=http://fileserver.pingcap.net POD_CONTAINER=golang GOPATH=/go BUILD_NUMBER=3484 USE_BAZEL_VERSION=6.5.0 KUBERNETES_PORT=tcp://10.233.0.1:443 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ HOME=/home/jenkins NODE_NAME=pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-2xgw-gnw10 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 JENKINS_HOME=/var/jenkins_home JOB_NAME=pingcap/tiflow/pull_cdc_integration_pulsar_test RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/3484/display/redirect?page=tests KUBERNETES_SERVICE_PORT_HTTPS=443 KUBERNETES_PORT_443_TCP_PORT=443 GIT_PREVIOUS_COMMIT=3e31ec89ba643faa364a78dbc7f062bb55f9c486 HUDSON_HOME=/var/jenkins_home JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/display/redirect PROW_JOB_ID=b4bfd23f-f137-430c-9c71-ddcc89dd7a6d TEST_GROUP=G14 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/3484/display/redirect?page=artifacts BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/3484/ TERM=xterm STAGE_NAME=Test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/ BUILD_DISPLAY_NAME=#3484 SHLVL=5 GIT_BRANCH=origin/main BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-3484 KUBERNETES_SERVICE_PORT=443 NODE_LABELS=pingcap_tiflow_pull_cdc_integration_pulsar_test_3484-2xgwq pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-2xgw-gnw10 PATH=/go/bin:/root/.cargo/bin:/usr/local/go/bin/:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/gradle-7.4.2/bin:/opt/apache-maven-3.8.8/bin:/usr/local/pulsar/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../../scripts/bin GIT_PREVIOUS_SUCCESSFUL_COMMIT=3e31ec89ba643faa364a78dbc7f062bb55f9c486 KUBERNETES_SERVICE_HOST=10.233.0.1 JENKINS_SERVER_COOKIE=durable-1c247575f7a5458f3a8df09baf43395940069f8be33f18a8fa953544ffab3376 _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/changefeed_finish/run.sh using Sink-Type: pulsar... <<================= The 1 times to try to start tidb cluster... Verifying downstream PD is started... public/default ++ date + echo '[Fri Apr 4 15:33:49 CST 2025] <<<<<< pulsar is ready >>>>>>' [Fri Apr 4 15:33:49 CST 2025] <<<<<< pulsar is ready >>>>>> + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_puller_lag.cli.6641.out cli changefeed create --start-ts=457114129117741057 '--sink-uri=pulsar://127.0.0.1:6650/ticdc-ddl-puller-lag-test-12569?protocol=canal-json&enable-tidb-extension=true' check diff failed 2-th time, retry later + set +x + tso='457114131712180235 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 457114131712180235 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/capture_session_done_during_task/run.sh using Sink-Type: pulsar... <<================= The 1 times to try to start tidb cluster... check diff failed 3-th time, retry later Create changefeed successfully! ID: 0a60beaf-b2b6-4aae-a937-c2dd14a8a510 Info: {"upstream_id":7489357859129328567,"namespace":"default","id":"0a60beaf-b2b6-4aae-a937-c2dd14a8a510","sink_uri":"pulsar://127.0.0.1:6650/ticdc-ddl-puller-lag-test-12569?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2025-04-04T15:33:50.27285412+08:00","start_ts":457114129117741057,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"pulsar_config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-10-ga306f4894","resolved_ts":457114129117741057,"checkpoint_ts":457114129117741057,"checkpoint_time":"2025-04-04 15:33:38.424"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... table sink_retry.finish_mark_2 exists check diff successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 4 15:33:48 CST 2025] <<<<<< run test case sink_retry success! >>>>>> find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/batch_add_table/run.sh using Sink-Type: pulsar... <<================= The 1 times to try to start tidb cluster... [Fri Apr 4 15:33:51 CST 2025] <<<<<< START cdc server in split_region case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.split_region.63526354.out server --log-file /tmp/tidb_cdc_test/split_region/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/split_region/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 The 1 times to try to start tidb cluster... + set +x [Fri Apr 4 15:33:51 CST 2025] <<<<<< START Pulsar consumer in ddl_puller_lag case >>>>>> Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release check diff failed 3-th time, retry later start tidb cluster in /tmp/tidb_cdc_test/simple Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... check diff failed 4-th time, retry later start tidb cluster in /tmp/tidb_cdc_test/owner_resign Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/tiflash Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/changefeed_finish Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... Verifying downstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/capture_session_done_during_task Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/processor_resolved_ts_fallback Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 4-th time, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release check diff successfully + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 07:33:54 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [817 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e8b9ca04-4d7b-47ec-8e7d-837b386ca160 {"id":"e8b9ca04-4d7b-47ec-8e7d-837b386ca160","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752031} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffb83c15ff e8b9ca04-4d7b-47ec-8e7d-837b386ca160 /tidb/cdc/default/default/upstream/7489357902462835193 {"id":7489357902462835193,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e8b9ca04-4d7b-47ec-8e7d-837b386ca160 {"id":"e8b9ca04-4d7b-47ec-8e7d-837b386ca160","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752031} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffb83c15ff e8b9ca04-4d7b-47ec-8e7d-837b386ca160 /tidb/cdc/default/default/upstream/7489357902462835193 {"id":7489357902462835193,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e8b9ca04-4d7b-47ec-8e7d-837b386ca160 {"id":"e8b9ca04-4d7b-47ec-8e7d-837b386ca160","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752031} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffb83c15ff e8b9ca04-4d7b-47ec-8e7d-837b386ca160 /tidb/cdc/default/default/upstream/7489357902462835193 {"id":7489357902462835193,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x ++ date + echo '[Fri Apr 4 15:33:54 CST 2025] <<<<<< START pulsar cluster in split_region case >>>>>>' [Fri Apr 4 15:33:54 CST 2025] <<<<<< START pulsar cluster in split_region case >>>>>> + workdir=/tmp/tidb_cdc_test/split_region + cluster_type=oauth + cd /tmp/tidb_cdc_test/split_region + DEFAULT_PULSAR_HOME=/usr/local/pulsar + pulsar_dir=/usr/local/pulsar ++ cat + mtls_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderTls brokerClientTlsEnabled=true brokerClientTrustCertsFilePath=/tmp/tidb_cdc_test/split_region/ca.cert.pem brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls brokerClientAuthenticationParameters={"tlsCertFile":"/tmp/tidb_cdc_test/split_region/broker_client.cert.pem","tlsKeyFile":"/tmp/tidb_cdc_test/split_region/broker_client.key-pk8.pem"} brokerServicePortTls=6651 webServicePortTls=8443 tlsTrustCertsFilePath=/tmp/tidb_cdc_test/split_region/ca.cert.pem tlsCertificateFilePath=/tmp/tidb_cdc_test/split_region/server.cert.pem tlsKeyFilePath=/tmp/tidb_cdc_test/split_region/server.key-pk8.pem tlsRequireTrustedClientCertOnConnect=true tlsAllowInsecureConnection=false tlsCertRefreshCheckDurationSec=300' ++ cat + normal_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' ++ cat + mtls_client_conf=' webServiceUrl=https://localhost:8443/ brokerServiceUrl=pulsar+ssl://localhost:6651/ authPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls authParams=tlsCertFile:/tmp/tidb_cdc_test/split_region/broker_client.cert.pem,tlsKeyFile:/tmp/tidb_cdc_test/split_region/broker_client.key-pk8.pem tlsTrustCertsFilePath=/tmp/tidb_cdc_test/split_region/ca.cert.pem' ++ cat + oauth_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/ authPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 authParams={"privateKey":"/tmp/tidb_cdc_test/split_region/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"}' ++ cat + oauth_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderToken brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 brokerClientAuthenticationParameters={"privateKey":"file:///tmp/tidb_cdc_test/split_region/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"} tokenSecretKey=data:;base64,U0poWDM2X0thcFlTeWJCdEpxMzVseFhfQnJyNExSVVJTa203UW1YSkdteThwVUZXOUVJT2NWUVBzeWt6OS1qag==' ++ cat + credential_json=' { "client_id":"1234", "client_secret":"e0KVlA2EiBfjoN13olyZd2kv1KL", "audience":"cdc-api-uri", "issuer_url":"http://localhost:9096", "type": "client_credentials" }' ++ cat + cert_server_conf='[ req ] default_bits = 2048 prompt = no default_md = sha256 distinguished_name = dn [ v3_ext ] authorityKeyIdentifier=keyid,issuer:always basicConstraints=CA:FALSE keyUsage=critical, digitalSignature, keyEncipherment extendedKeyUsage=serverAuth subjectAltName=@alt_names [ dn ] CN = server [ alt_names ] DNS.1 = localhost IP.1 = 127.0.0.1' + echo ' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' + cp /usr/local/pulsar/conf/standalone.conf /tmp/tidb_cdc_test/split_region/pulsar_standalone.conf + pulsar_port=6650 + '[' oauth == mtls ']' + '[' oauth == oauth ']' + echo 'Waiting for oauth2 server to be ready...' Waiting for oauth2 server to be ready... + i=0 + nc -z localhost 9096 + oauth2-server + gen_oauth_config + echo ' { "client_id":"1234", "client_secret":"e0KVlA2EiBfjoN13olyZd2kv1KL", "audience":"cdc-api-uri", "issuer_url":"http://localhost:9096", "type": "client_credentials" }' + echo ' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderToken brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 brokerClientAuthenticationParameters={"privateKey":"file:///tmp/tidb_cdc_test/split_region/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"} tokenSecretKey=data:;base64,U0poWDM2X0thcFlTeWJCdEpxMzVseFhfQnJyNExSVVJTa203UW1YSkdteThwVUZXOUVJT2NWUVBzeWt6OS1qag==' + echo ' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/ authPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 authParams={"privateKey":"/tmp/tidb_cdc_test/split_region/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"}' ++ date + echo '[Fri Apr 4 15:33:54 CST 2025] <<<<<< START pulsar cluster in oauth mode in split_region case >>>>>>' [Fri Apr 4 15:33:54 CST 2025] <<<<<< START pulsar cluster in oauth mode in split_region case >>>>>> + echo 'Waiting for pulsar port to be ready...' Waiting for pulsar port to be ready... + i=0 + nc -z localhost 6650 + /usr/local/pulsar/bin/pulsar standalone --config /tmp/tidb_cdc_test/split_region/pulsar_standalone.conf -nfw --metadata-dir /tmp/tidb_cdc_test/split_region/pulsar-metadata --bookkeeper-dir /tmp/tidb_cdc_test/split_region/pulsar-bookie + i=1 + '[' 1 -gt 20 ']' + sleep 2 start tidb cluster in /tmp/tidb_cdc_test/resolve_lock Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/batch_add_table Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release check diff failed 1-th time, retry later Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiDB... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489357935344335316 TiDB Cluster ID. tikv_gc_leader_uuid 657fee1ee54000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-0lxq-6vnt9, pid:6124, start at 2025-04-04 15:33:55.241228751 +0800 CST m=+1.616967818 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:35:55.254 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:33:55.221 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:23:55.221 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489357935344335316 TiDB Cluster ID. tikv_gc_leader_uuid 657fee1ee54000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-0lxq-6vnt9, pid:6124, start at 2025-04-04 15:33:55.241228751 +0800 CST m=+1.616967818 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:35:55.254 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:33:55.221 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:23:55.221 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489357943628639486 TiDB Cluster ID. tikv_gc_leader_uuid 657fee1ef04000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-0lxq-6vnt9, pid:6212, start at 2025-04-04 15:33:55.29988029 +0800 CST m=+1.603441596 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:35:55.309 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:33:55.315 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:23:55.315 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 5-th time, retry later Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/canal_json_content_compatible/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/canal_json_content_compatible/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/canal_json_content_compatible/tiflash/db/proxy"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/canal_json_content_compatible/tiflash/log/proxy.log"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/canal_json_content_compatible/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release [Pipeline] // stage + nc -z localhost 6650 + i=2 + '[' 2 -gt 20 ']' + sleep 2 [Pipeline] } [Pipeline] // stage Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Verifying downstream PD is started... Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 2-th time, retry later [Fri Apr 4 15:33:58 CST 2025] <<<<<< START cdc server in canal_json_content_compatible case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.canal_json_content_compatible.76037605.out server --log-file /tmp/tidb_cdc_test/canal_json_content_compatible/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/canal_json_content_compatible/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/multi_capture/run.sh using Sink-Type: pulsar... <<================= The 1 times to try to start tidb cluster... [Pipeline] } [Pipeline] // container [Pipeline] } Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489357950171925030 TiDB Cluster ID. tikv_gc_leader_uuid 657fee210f80008 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-7fxb-ntnhx, pid:1493, start at 2025-04-04 15:33:57.451643551 +0800 CST m=+1.578310620 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:35:57.460 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:33:57.438 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:23:57.438 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489357950171925030 TiDB Cluster ID. tikv_gc_leader_uuid 657fee210f80008 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-7fxb-ntnhx, pid:1493, start at 2025-04-04 15:33:57.451643551 +0800 CST m=+1.578310620 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:35:57.460 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:33:57.438 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:23:57.438 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489357952010738355 TiDB Cluster ID. tikv_gc_leader_uuid 657fee211b4000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-7fxb-ntnhx, pid:1542, start at 2025-04-04 15:33:57.502988558 +0800 CST m=+1.567681189 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:35:57.510 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:33:57.485 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:23:57.485 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/owner_resign/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/owner_resign/tiflash/log/error.log arg matches is ArgMatches { args: {"pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/owner_resign/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/owner_resign/tiflash/log/proxy.log"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/owner_resign/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } check diff failed 6-th time, retry later [Pipeline] // container [Pipeline] } [Pipeline] // withEnv Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + nc -z localhost 6650 + echo 'Waiting for pulsar namespace to be ready...' Waiting for pulsar namespace to be ready... + i=0 + /usr/local/pulsar/bin/pulsar-admin namespaces list public [Pipeline] } [Pipeline] // withEnv [Pipeline] } Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/changefeed_error/run.sh using Sink-Type: pulsar... <<================= The 1 times to try to start tidb cluster... [Pipeline] // node [Pipeline] } VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489357955358834369 TiDB Cluster ID. tikv_gc_leader_uuid 657fee21e94000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-vgqp-h2d14, pid:1349, start at 2025-04-04 15:33:58.354802853 +0800 CST m=+1.614565254 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:35:58.362 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:33:58.359 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:23:58.359 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489357955358834369 TiDB Cluster ID. tikv_gc_leader_uuid 657fee21e94000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-vgqp-h2d14, pid:1349, start at 2025-04-04 15:33:58.354802853 +0800 CST m=+1.614565254 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:35:58.362 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:33:58.359 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:23:58.359 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489357957044799713 TiDB Cluster ID. tikv_gc_leader_uuid 657fee21f50000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-vgqp-h2d14, pid:1425, start at 2025-04-04 15:33:58.381509838 +0800 CST m=+1.573600280 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:35:58.390 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:33:58.356 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:23:58.356 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/tiflash/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/tiflash/tiflash/log/error.log [Pipeline] // node [Pipeline] } arg matches is ArgMatches { args: {"engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/tiflash/tiflash/db/proxy"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/tiflash/tiflash-proxy.toml"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/tiflash/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [Pipeline] // podTemplate [Pipeline] } VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489357950984869366 TiDB Cluster ID. tikv_gc_leader_uuid 657fee226780009 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-b84s-w4c6p, pid:1425, start at 2025-04-04 15:33:58.827647031 +0800 CST m=+1.544766331 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:35:58.835 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:33:58.814 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:23:58.814 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489357950984869366 TiDB Cluster ID. tikv_gc_leader_uuid 657fee226780009 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-b84s-w4c6p, pid:1425, start at 2025-04-04 15:33:58.827647031 +0800 CST m=+1.544766331 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:35:58.835 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:33:58.814 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:23:58.814 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489357961577662623 TiDB Cluster ID. tikv_gc_leader_uuid 657fee226780003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-b84s-w4c6p, pid:1502, start at 2025-04-04 15:33:58.817530671 +0800 CST m=+1.465673624 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:35:58.824 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:33:58.814 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:23:58.814 +0800 All versions after safe point can be accessed. (DO NOT EDIT) [Pipeline] // podTemplate [Pipeline] } Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489357962550911928 TiDB Cluster ID. tikv_gc_leader_uuid 657fee226580002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-5mln-gcmj2, pid:1429, start at 2025-04-04 15:33:58.808005756 +0800 CST m=+1.478854899 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:35:58.815 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:33:58.806 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:23:58.806 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489357962550911928 TiDB Cluster ID. tikv_gc_leader_uuid 657fee226580002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-5mln-gcmj2, pid:1429, start at 2025-04-04 15:33:58.808005756 +0800 CST m=+1.478854899 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:35:58.815 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:33:58.806 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:23:58.806 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489357955433776585 TiDB Cluster ID. tikv_gc_leader_uuid 657fee22658000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-5mln-gcmj2, pid:1496, start at 2025-04-04 15:33:58.849104474 +0800 CST m=+1.444992888 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:35:58.856 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:33:58.857 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:23:58.857 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/processor_resolved_ts_fallback/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/processor_resolved_ts_fallback/tiflash/log/error.log [Pipeline] // withEnv [Pipeline] } Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/simple/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/simple/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/simple/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/simple/tiflash-proxy.toml"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/simple/tiflash/log/proxy.log"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [Pipeline] // withEnv [Pipeline] } check diff failed 3-th time, retry later arg matches is ArgMatches { args: {"memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/processor_resolved_ts_fallback/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/processor_resolved_ts_fallback/tiflash-proxy.toml"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/processor_resolved_ts_fallback/tiflash/log/proxy.log"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [Pipeline] // stage [Pipeline] } [Pipeline] // stage [Pipeline] } VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489357962719330633 TiDB Cluster ID. tikv_gc_leader_uuid 657fee235d80012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-qj98-cgzmv, pid:1519, start at 2025-04-04 15:33:59.835639496 +0800 CST m=+1.608444631 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:35:59.844 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:33:59.848 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:23:59.848 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489357962719330633 TiDB Cluster ID. tikv_gc_leader_uuid 657fee235d80012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-qj98-cgzmv, pid:1519, start at 2025-04-04 15:33:59.835639496 +0800 CST m=+1.608444631 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:35:59.844 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:33:59.848 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:23:59.848 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489357966580177035 TiDB Cluster ID. tikv_gc_leader_uuid 657fee235d40003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-qj98-cgzmv, pid:1577, start at 2025-04-04 15:33:59.800041566 +0800 CST m=+1.508097245 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:35:59.807 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:33:59.797 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:23:59.797 +0800 All versions after safe point can be accessed. (DO NOT EDIT) [Fri Apr 4 15:34:00 CST 2025] <<<<<< START cdc server in owner_resign case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.owner_resign.28612863.out server --log-file /tmp/tidb_cdc_test/owner_resign/cdc1.log --log-level debug --data-dir /tmp/tidb_cdc_test/owner_resign/cdc_data1 --cluster-id default --addr 127.0.0.1:8301 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8301 failed: Connection refused * Failed to connect to 127.0.0.1 port 8301: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff successfully VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489357966190138613 TiDB Cluster ID. tikv_gc_leader_uuid 657fee238b0000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-2xgw-gnw10, pid:1337, start at 2025-04-04 15:34:00.026403419 +0800 CST m=+1.562589732 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:36:00.039 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:34:00.030 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:24:00.030 +0800 All versions after safe point can be accessed. (DO NOT EDIT) public/default Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/capture_session_done_during_task/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/capture_session_done_during_task/tiflash/log/error.log arg matches is ArgMatches { args: {"log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/capture_session_done_during_task/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/capture_session_done_during_task/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/capture_session_done_during_task/tiflash/db/proxy"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } wait process cdc.test exit for 1-th time... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489357966190138613 TiDB Cluster ID. tikv_gc_leader_uuid 657fee238b0000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-2xgw-gnw10, pid:1337, start at 2025-04-04 15:34:00.026403419 +0800 CST m=+1.562589732 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:36:00.039 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:34:00.030 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:24:00.030 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489357952478745690 TiDB Cluster ID. tikv_gc_leader_uuid 657fee23b28000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-2xgw-gnw10, pid:1410, start at 2025-04-04 15:34:00.184949759 +0800 CST m=+1.649823311 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:36:00.192 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:34:00.188 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:24:00.188 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Logging trace to /tmp/tidb_cdc_test/changefeed_finish/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/changefeed_finish/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/changefeed_finish/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/changefeed_finish/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/changefeed_finish/tiflash/log/proxy.log"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } ++ date + echo '[Fri Apr 4 15:34:00 CST 2025] <<<<<< pulsar is ready >>>>>>' [Fri Apr 4 15:34:00 CST 2025] <<<<<< pulsar is ready >>>>>> + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.split_region.cli.6830.out cli changefeed create --start-ts=457114131712180235 '--sink-uri=pulsar://127.0.0.1:6650/ticdc-split-region-test-6355?protocol=canal-json&enable-tidb-extension=true' --config /tmp/tidb_cdc_test/split_region/pulsar_test.toml + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.tiflash.cli.2873.out cli tso query --pd=http://127.0.0.1:2379 wait process cdc.test exit for 2-th time... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 07:34:01 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [817 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7227e7c6-a133-41ec-8403-7d36753321b9 {"id":"7227e7c6-a133-41ec-8403-7d36753321b9","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752038} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffb86130d4 7227e7c6-a133-41ec-8403-7d36753321b9 /tidb/cdc/default/default/upstream/7489357935344335316 {"id":7489357935344335316,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7227e7c6-a133-41ec-8403-7d36753321b9 {"id":"7227e7c6-a133-41ec-8403-7d36753321b9","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752038} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffb86130d4 7227e7c6-a133-41ec-8403-7d36753321b9 /tidb/cdc/default/default/upstream/7489357935344335316 {"id":7489357935344335316,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7227e7c6-a133-41ec-8403-7d36753321b9 {"id":"7227e7c6-a133-41ec-8403-7d36753321b9","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752038} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffb86130d4 7227e7c6-a133-41ec-8403-7d36753321b9 /tidb/cdc/default/default/upstream/7489357935344335316 {"id":7489357935344335316,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x ++ date + echo '[Fri Apr 4 15:34:01 CST 2025] <<<<<< START pulsar cluster in canal_json_content_compatible case >>>>>>' [Fri Apr 4 15:34:01 CST 2025] <<<<<< START pulsar cluster in canal_json_content_compatible case >>>>>> + workdir=/tmp/tidb_cdc_test/canal_json_content_compatible + cluster_type=normal + cd /tmp/tidb_cdc_test/canal_json_content_compatible + DEFAULT_PULSAR_HOME=/usr/local/pulsar + pulsar_dir=/usr/local/pulsar ++ cat + mtls_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderTls brokerClientTlsEnabled=true brokerClientTrustCertsFilePath=/tmp/tidb_cdc_test/canal_json_content_compatible/ca.cert.pem brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls brokerClientAuthenticationParameters={"tlsCertFile":"/tmp/tidb_cdc_test/canal_json_content_compatible/broker_client.cert.pem","tlsKeyFile":"/tmp/tidb_cdc_test/canal_json_content_compatible/broker_client.key-pk8.pem"} brokerServicePortTls=6651 webServicePortTls=8443 tlsTrustCertsFilePath=/tmp/tidb_cdc_test/canal_json_content_compatible/ca.cert.pem tlsCertificateFilePath=/tmp/tidb_cdc_test/canal_json_content_compatible/server.cert.pem tlsKeyFilePath=/tmp/tidb_cdc_test/canal_json_content_compatible/server.key-pk8.pem tlsRequireTrustedClientCertOnConnect=true tlsAllowInsecureConnection=false tlsCertRefreshCheckDurationSec=300' ++ cat + normal_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' ++ cat + mtls_client_conf=' webServiceUrl=https://localhost:8443/ brokerServiceUrl=pulsar+ssl://localhost:6651/ authPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls authParams=tlsCertFile:/tmp/tidb_cdc_test/canal_json_content_compatible/broker_client.cert.pem,tlsKeyFile:/tmp/tidb_cdc_test/canal_json_content_compatible/broker_client.key-pk8.pem tlsTrustCertsFilePath=/tmp/tidb_cdc_test/canal_json_content_compatible/ca.cert.pem' ++ cat + oauth_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/ authPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 authParams={"privateKey":"/tmp/tidb_cdc_test/canal_json_content_compatible/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"}' ++ cat + oauth_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderToken brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 brokerClientAuthenticationParameters={"privateKey":"file:///tmp/tidb_cdc_test/canal_json_content_compatible/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"} tokenSecretKey=data:;base64,U0poWDM2X0thcFlTeWJCdEpxMzVseFhfQnJyNExSVVJTa203UW1YSkdteThwVUZXOUVJT2NWUVBzeWt6OS1qag==' ++ cat + credential_json=' { "client_id":"1234", "client_secret":"e0KVlA2EiBfjoN13olyZd2kv1KL", "audience":"cdc-api-uri", "issuer_url":"http://localhost:9096", "type": "client_credentials" }' ++ cat + cert_server_conf='[ req ] default_bits = 2048 prompt = no default_md = sha256 distinguished_name = dn [ v3_ext ] authorityKeyIdentifier=keyid,issuer:always basicConstraints=CA:FALSE keyUsage=critical, digitalSignature, keyEncipherment extendedKeyUsage=serverAuth subjectAltName=@alt_names [ dn ] CN = server [ alt_names ] DNS.1 = localhost IP.1 = 127.0.0.1' + echo ' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' + cp /usr/local/pulsar/conf/standalone.conf /tmp/tidb_cdc_test/canal_json_content_compatible/pulsar_standalone.conf + pulsar_port=6650 + '[' normal == mtls ']' + '[' normal == oauth ']' + echo 'no cluster type specified, using default configuration.' no cluster type specified, using default configuration. ++ date + echo '[Fri Apr 4 15:34:01 CST 2025] <<<<<< START pulsar cluster in normal mode in canal_json_content_compatible case >>>>>>' [Fri Apr 4 15:34:01 CST 2025] <<<<<< START pulsar cluster in normal mode in canal_json_content_compatible case >>>>>> + echo 'Waiting for pulsar port to be ready...' Waiting for pulsar port to be ready... + i=0 + nc -z localhost 6650 + /usr/local/pulsar/bin/pulsar standalone --config /tmp/tidb_cdc_test/canal_json_content_compatible/pulsar_standalone.conf -nfw --metadata-dir /tmp/tidb_cdc_test/canal_json_content_compatible/pulsar-metadata --bookkeeper-dir /tmp/tidb_cdc_test/canal_json_content_compatible/pulsar-bookie + i=1 + '[' 1 -gt 20 ']' + sleep 2 start tidb cluster in /tmp/tidb_cdc_test/multi_capture Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489357969709569958 TiDB Cluster ID. tikv_gc_leader_uuid 657fee24410000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-6nlr-p88d4, pid:1552, start at 2025-04-04 15:34:00.733156035 +0800 CST m=+1.496419714 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:36:00.741 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:34:00.708 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:24:00.708 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489357969709569958 TiDB Cluster ID. tikv_gc_leader_uuid 657fee24410000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-6nlr-p88d4, pid:1552, start at 2025-04-04 15:34:00.733156035 +0800 CST m=+1.496419714 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:36:00.741 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:34:00.708 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:24:00.708 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489357969074274869 TiDB Cluster ID. tikv_gc_leader_uuid 657fee24590000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-6nlr-p88d4, pid:1622, start at 2025-04-04 15:34:00.83216172 +0800 CST m=+1.526172098 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:36:00.841 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:34:00.804 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:24:00.804 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... check diff failed 4-th time, retry later [Fri Apr 4 15:34:01 CST 2025] <<<<<< START cdc server in processor_resolved_ts_fallback case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sink/dmlsink/txn/mysql/MySQLSinkExecDMLError=return(true);github.com/pingcap/tiflow/cdc/sink/dmlsink/mq/dmlproducer/KafkaSinkAsyncSendError=return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.processor_resolved_ts_fallback.28712873.out server --log-file /tmp/tidb_cdc_test/processor_resolved_ts_fallback/cdc1.log --log-level debug --data-dir /tmp/tidb_cdc_test/processor_resolved_ts_fallback/cdc_data1 --cluster-id default --addr 127.0.0.1:8301 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8301 failed: Connection refused * Failed to connect to 127.0.0.1 port 8301: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 wait process 6161 exit for 1-th time... wait process 6161 exit for 2-th time... wait process 6161 exit for 3-th time... wait process 6161 exit for 4-th time... wait process 6161 exit for 5-th time... wait process 6161 exit for 6-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 4 15:34:01 CST 2025] <<<<<< run test case processor_stop_delay success! >>>>>> Logging trace to /tmp/tidb_cdc_test/resolve_lock/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/resolve_lock/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/resolve_lock/tiflash-proxy.toml"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/resolve_lock/tiflash/log/proxy.log"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/resolve_lock/tiflash/db/proxy"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } wait process 6161 exit for 7-th time... Create changefeed successfully! ID: 68e00efa-4134-4b45-87df-be63d3d54ebd Info: {"upstream_id":7489357902462835193,"namespace":"default","id":"68e00efa-4134-4b45-87df-be63d3d54ebd","sink_uri":"pulsar://127.0.0.1:6650/ticdc-split-region-test-6355?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2025-04-04T15:34:01.894453556+08:00","start_ts":457114131712180235,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"pulsar_config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30,"oauth2":{"oauth2-issuer-url":"http://localhost:9096","oauth2-audience":"cdc-api-uri","oauth2-private-key":"/tmp/tidb_cdc_test/split_region/credential.json","oauth2-client-id":"1234"}},"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-10-ga306f4894","resolved_ts":457114131712180235,"checkpoint_ts":457114131712180235,"checkpoint_time":"2025-04-04 15:33:48.321"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.simple.cli.2884.out cli tso query --pd=http://127.0.0.1:2379 start tidb cluster in /tmp/tidb_cdc_test/changefeed_error Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... wait process 6161 exit for 8-th time... ++ date + echo '[Fri Apr 4 15:34:02 CST 2025] <<<<<< START pulsar cluster in changefeed_finish case >>>>>>' [Fri Apr 4 15:34:02 CST 2025] <<<<<< START pulsar cluster in changefeed_finish case >>>>>> + workdir=/tmp/tidb_cdc_test/changefeed_finish + cluster_type=normal + cd /tmp/tidb_cdc_test/changefeed_finish + DEFAULT_PULSAR_HOME=/usr/local/pulsar + pulsar_dir=/usr/local/pulsar ++ cat + mtls_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderTls brokerClientTlsEnabled=true brokerClientTrustCertsFilePath=/tmp/tidb_cdc_test/changefeed_finish/ca.cert.pem brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls brokerClientAuthenticationParameters={"tlsCertFile":"/tmp/tidb_cdc_test/changefeed_finish/broker_client.cert.pem","tlsKeyFile":"/tmp/tidb_cdc_test/changefeed_finish/broker_client.key-pk8.pem"} brokerServicePortTls=6651 webServicePortTls=8443 tlsTrustCertsFilePath=/tmp/tidb_cdc_test/changefeed_finish/ca.cert.pem tlsCertificateFilePath=/tmp/tidb_cdc_test/changefeed_finish/server.cert.pem tlsKeyFilePath=/tmp/tidb_cdc_test/changefeed_finish/server.key-pk8.pem tlsRequireTrustedClientCertOnConnect=true tlsAllowInsecureConnection=false tlsCertRefreshCheckDurationSec=300' ++ cat + normal_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' ++ cat + mtls_client_conf=' webServiceUrl=https://localhost:8443/ brokerServiceUrl=pulsar+ssl://localhost:6651/ authPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls authParams=tlsCertFile:/tmp/tidb_cdc_test/changefeed_finish/broker_client.cert.pem,tlsKeyFile:/tmp/tidb_cdc_test/changefeed_finish/broker_client.key-pk8.pem tlsTrustCertsFilePath=/tmp/tidb_cdc_test/changefeed_finish/ca.cert.pem' ++ cat + oauth_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/ authPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 authParams={"privateKey":"/tmp/tidb_cdc_test/changefeed_finish/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"}' ++ cat + oauth_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderToken brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 brokerClientAuthenticationParameters={"privateKey":"file:///tmp/tidb_cdc_test/changefeed_finish/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"} tokenSecretKey=data:;base64,U0poWDM2X0thcFlTeWJCdEpxMzVseFhfQnJyNExSVVJTa203UW1YSkdteThwVUZXOUVJT2NWUVBzeWt6OS1qag==' ++ cat + credential_json=' { "client_id":"1234", "client_secret":"e0KVlA2EiBfjoN13olyZd2kv1KL", "audience":"cdc-api-uri", "issuer_url":"http://localhost:9096", "type": "client_credentials" }' ++ cat + cert_server_conf='[ req ] default_bits = 2048 prompt = no default_md = sha256 distinguished_name = dn [ v3_ext ] authorityKeyIdentifier=keyid,issuer:always basicConstraints=CA:FALSE keyUsage=critical, digitalSignature, keyEncipherment extendedKeyUsage=serverAuth subjectAltName=@alt_names [ dn ] CN = server [ alt_names ] DNS.1 = localhost IP.1 = 127.0.0.1' + echo ' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' + cp /usr/local/pulsar/conf/standalone.conf /tmp/tidb_cdc_test/changefeed_finish/pulsar_standalone.conf + pulsar_port=6650 + '[' normal == mtls ']' + '[' normal == oauth ']' + echo 'no cluster type specified, using default configuration.' no cluster type specified, using default configuration. ++ date + echo '[Fri Apr 4 15:34:02 CST 2025] <<<<<< START pulsar cluster in normal mode in changefeed_finish case >>>>>>' [Fri Apr 4 15:34:02 CST 2025] <<<<<< START pulsar cluster in normal mode in changefeed_finish case >>>>>> + echo 'Waiting for pulsar port to be ready...' Waiting for pulsar port to be ready... + i=0 + nc -z localhost 6650 + /usr/local/pulsar/bin/pulsar standalone --config /tmp/tidb_cdc_test/changefeed_finish/pulsar_standalone.conf -nfw --metadata-dir /tmp/tidb_cdc_test/changefeed_finish/pulsar-metadata --bookkeeper-dir /tmp/tidb_cdc_test/changefeed_finish/pulsar-bookie + i=1 + '[' 1 -gt 20 ']' + sleep 2 + set +x + tso='457114135156228097 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 457114135156228097 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 4 15:34:02 CST 2025] <<<<<< START cdc server in tiflash case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.tiflash.29012903.out server --log-file /tmp/tidb_cdc_test/tiflash/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/tiflash/cdc_data --cluster-id default + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ++ date + echo '[Fri Apr 4 15:34:02 CST 2025] <<<<<< START pulsar cluster in capture_session_done_during_task case >>>>>>' [Fri Apr 4 15:34:02 CST 2025] <<<<<< START pulsar cluster in capture_session_done_during_task case >>>>>> + workdir=/tmp/tidb_cdc_test/capture_session_done_during_task + cluster_type=normal + cd /tmp/tidb_cdc_test/capture_session_done_during_task + DEFAULT_PULSAR_HOME=/usr/local/pulsar + pulsar_dir=/usr/local/pulsar ++ cat + mtls_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderTls brokerClientTlsEnabled=true brokerClientTrustCertsFilePath=/tmp/tidb_cdc_test/capture_session_done_during_task/ca.cert.pem brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls brokerClientAuthenticationParameters={"tlsCertFile":"/tmp/tidb_cdc_test/capture_session_done_during_task/broker_client.cert.pem","tlsKeyFile":"/tmp/tidb_cdc_test/capture_session_done_during_task/broker_client.key-pk8.pem"} brokerServicePortTls=6651 webServicePortTls=8443 tlsTrustCertsFilePath=/tmp/tidb_cdc_test/capture_session_done_during_task/ca.cert.pem tlsCertificateFilePath=/tmp/tidb_cdc_test/capture_session_done_during_task/server.cert.pem tlsKeyFilePath=/tmp/tidb_cdc_test/capture_session_done_during_task/server.key-pk8.pem tlsRequireTrustedClientCertOnConnect=true tlsAllowInsecureConnection=false tlsCertRefreshCheckDurationSec=300' ++ cat + normal_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' ++ cat + mtls_client_conf=' webServiceUrl=https://localhost:8443/ brokerServiceUrl=pulsar+ssl://localhost:6651/ authPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls authParams=tlsCertFile:/tmp/tidb_cdc_test/capture_session_done_during_task/broker_client.cert.pem,tlsKeyFile:/tmp/tidb_cdc_test/capture_session_done_during_task/broker_client.key-pk8.pem tlsTrustCertsFilePath=/tmp/tidb_cdc_test/capture_session_done_during_task/ca.cert.pem' ++ cat + oauth_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/ authPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 authParams={"privateKey":"/tmp/tidb_cdc_test/capture_session_done_during_task/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"}' ++ cat + oauth_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderToken brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 brokerClientAuthenticationParameters={"privateKey":"file:///tmp/tidb_cdc_test/capture_session_done_during_task/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"} tokenSecretKey=data:;base64,U0poWDM2X0thcFlTeWJCdEpxMzVseFhfQnJyNExSVVJTa203UW1YSkdteThwVUZXOUVJT2NWUVBzeWt6OS1qag==' ++ cat + credential_json=' { "client_id":"1234", "client_secret":"e0KVlA2EiBfjoN13olyZd2kv1KL", "audience":"cdc-api-uri", "issuer_url":"http://localhost:9096", "type": "client_credentials" }' ++ cat + cert_server_conf='[ req ] default_bits = 2048 prompt = no default_md = sha256 distinguished_name = dn [ v3_ext ] authorityKeyIdentifier=keyid,issuer:always basicConstraints=CA:FALSE keyUsage=critical, digitalSignature, keyEncipherment extendedKeyUsage=serverAuth subjectAltName=@alt_names [ dn ] CN = server [ alt_names ] DNS.1 = localhost IP.1 = 127.0.0.1' + echo ' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' + cp /usr/local/pulsar/conf/standalone.conf /tmp/tidb_cdc_test/capture_session_done_during_task/pulsar_standalone.conf + pulsar_port=6650 + '[' normal == mtls ']' + '[' normal == oauth ']' + echo 'no cluster type specified, using default configuration.' no cluster type specified, using default configuration. ++ date + echo '[Fri Apr 4 15:34:02 CST 2025] <<<<<< START pulsar cluster in normal mode in capture_session_done_during_task case >>>>>>' [Fri Apr 4 15:34:02 CST 2025] <<<<<< START pulsar cluster in normal mode in capture_session_done_during_task case >>>>>> + echo 'Waiting for pulsar port to be ready...' Waiting for pulsar port to be ready... + i=0 + nc -z localhost 6650 + /usr/local/pulsar/bin/pulsar standalone --config /tmp/tidb_cdc_test/capture_session_done_during_task/pulsar_standalone.conf -nfw --metadata-dir /tmp/tidb_cdc_test/capture_session_done_during_task/pulsar-metadata --bookkeeper-dir /tmp/tidb_cdc_test/capture_session_done_during_task/pulsar-bookie + i=1 + '[' 1 -gt 20 ']' + sleep 2 wait process 6161 exit for 9-th time... + nc -z localhost 6650 + i=2 + '[' 2 -gt 20 ']' + sleep 2 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489357967512829777 TiDB Cluster ID. tikv_gc_leader_uuid 657fee25b0c000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-j3lw-c6tm2, pid:1566, start at 2025-04-04 15:34:02.219551546 +0800 CST m=+1.428799263 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:36:02.227 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:34:02.229 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:24:02.229 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489357967512829777 TiDB Cluster ID. tikv_gc_leader_uuid 657fee25b0c000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-j3lw-c6tm2, pid:1566, start at 2025-04-04 15:34:02.219551546 +0800 CST m=+1.428799263 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:36:02.227 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:34:02.229 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:24:02.229 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489357972288784530 TiDB Cluster ID. tikv_gc_leader_uuid 657fee25d5c000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-j3lw-c6tm2, pid:1646, start at 2025-04-04 15:34:02.345471121 +0800 CST m=+1.471855044 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:36:02.353 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:34:02.327 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:24:02.327 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/batch_add_table/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/batch_add_table/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/batch_add_table/tiflash/log/proxy.log"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/batch_add_table/tiflash-proxy.toml"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/batch_add_table/tiflash/db/proxy"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + set +x [Fri Apr 4 15:34:03 CST 2025] <<<<<< START Pulsar consumer in split_region case >>>>>> table split_region.test1 not exists for 1-th check, retry later + set +x + tso='457114135301718017 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 457114135301718017 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 4 15:34:03 CST 2025] <<<<<< START cdc server in simple case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.simple.29272929.out server --log-file /tmp/tidb_cdc_test/simple/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/simple/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8301 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 07:34:03 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [817 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/cd58ca7c-c620-4bf4-997f-0744ba25e64f {"id":"cd58ca7c-c620-4bf4-997f-0744ba25e64f","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752040} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffb86df3cc cd58ca7c-c620-4bf4-997f-0744ba25e64f /tidb/cdc/default/default/upstream/7489357950171925030 {"id":7489357950171925030,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/cd58ca7c-c620-4bf4-997f-0744ba25e64f {"id":"cd58ca7c-c620-4bf4-997f-0744ba25e64f","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752040} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffb86df3cc cd58ca7c-c620-4bf4-997f-0744ba25e64f /tidb/cdc/default/default/upstream/7489357950171925030 {"id":7489357950171925030,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/cd58ca7c-c620-4bf4-997f-0744ba25e64f {"id":"cd58ca7c-c620-4bf4-997f-0744ba25e64f","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752040} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffb86df3cc cd58ca7c-c620-4bf4-997f-0744ba25e64f /tidb/cdc/default/default/upstream/7489357950171925030 {"id":7489357950171925030,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x cdc cli capture list --server http://127.0.0.1:8301 |jq '.|length'|grep -E '^1$' 1 run task successfully ++ date + echo '[Fri Apr 4 15:34:03 CST 2025] <<<<<< START pulsar cluster in owner_resign case >>>>>>' [Fri Apr 4 15:34:03 CST 2025] <<<<<< START pulsar cluster in owner_resign case >>>>>> + workdir=/tmp/tidb_cdc_test/owner_resign + cluster_type=normal + cd /tmp/tidb_cdc_test/owner_resign + DEFAULT_PULSAR_HOME=/usr/local/pulsar + pulsar_dir=/usr/local/pulsar ++ cat + mtls_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderTls brokerClientTlsEnabled=true brokerClientTrustCertsFilePath=/tmp/tidb_cdc_test/owner_resign/ca.cert.pem brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls brokerClientAuthenticationParameters={"tlsCertFile":"/tmp/tidb_cdc_test/owner_resign/broker_client.cert.pem","tlsKeyFile":"/tmp/tidb_cdc_test/owner_resign/broker_client.key-pk8.pem"} brokerServicePortTls=6651 webServicePortTls=8443 tlsTrustCertsFilePath=/tmp/tidb_cdc_test/owner_resign/ca.cert.pem tlsCertificateFilePath=/tmp/tidb_cdc_test/owner_resign/server.cert.pem tlsKeyFilePath=/tmp/tidb_cdc_test/owner_resign/server.key-pk8.pem tlsRequireTrustedClientCertOnConnect=true tlsAllowInsecureConnection=false tlsCertRefreshCheckDurationSec=300' ++ cat + normal_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' ++ cat + mtls_client_conf=' webServiceUrl=https://localhost:8443/ brokerServiceUrl=pulsar+ssl://localhost:6651/ authPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls authParams=tlsCertFile:/tmp/tidb_cdc_test/owner_resign/broker_client.cert.pem,tlsKeyFile:/tmp/tidb_cdc_test/owner_resign/broker_client.key-pk8.pem tlsTrustCertsFilePath=/tmp/tidb_cdc_test/owner_resign/ca.cert.pem' ++ cat + oauth_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/ authPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 authParams={"privateKey":"/tmp/tidb_cdc_test/owner_resign/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"}' ++ cat + oauth_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderToken brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 brokerClientAuthenticationParameters={"privateKey":"file:///tmp/tidb_cdc_test/owner_resign/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"} tokenSecretKey=data:;base64,U0poWDM2X0thcFlTeWJCdEpxMzVseFhfQnJyNExSVVJTa203UW1YSkdteThwVUZXOUVJT2NWUVBzeWt6OS1qag==' ++ cat + credential_json=' { "client_id":"1234", "client_secret":"e0KVlA2EiBfjoN13olyZd2kv1KL", "audience":"cdc-api-uri", "issuer_url":"http://localhost:9096", "type": "client_credentials" }' ++ cat + cert_server_conf='[ req ] default_bits = 2048 prompt = no default_md = sha256 distinguished_name = dn [ v3_ext ] authorityKeyIdentifier=keyid,issuer:always basicConstraints=CA:FALSE keyUsage=critical, digitalSignature, keyEncipherment extendedKeyUsage=serverAuth subjectAltName=@alt_names [ dn ] CN = server [ alt_names ] DNS.1 = localhost IP.1 = 127.0.0.1' + echo ' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' + cp /usr/local/pulsar/conf/standalone.conf /tmp/tidb_cdc_test/owner_resign/pulsar_standalone.conf + pulsar_port=6650 + '[' normal == mtls ']' + '[' normal == oauth ']' + echo 'no cluster type specified, using default configuration.' no cluster type specified, using default configuration. ++ date + echo '[Fri Apr 4 15:34:03 CST 2025] <<<<<< START pulsar cluster in normal mode in owner_resign case >>>>>>' [Fri Apr 4 15:34:03 CST 2025] <<<<<< START pulsar cluster in normal mode in owner_resign case >>>>>> + echo 'Waiting for pulsar port to be ready...' Waiting for pulsar port to be ready... + i=0 + /usr/local/pulsar/bin/pulsar standalone --config /tmp/tidb_cdc_test/owner_resign/pulsar_standalone.conf -nfw --metadata-dir /tmp/tidb_cdc_test/owner_resign/pulsar-metadata --bookkeeper-dir /tmp/tidb_cdc_test/owner_resign/pulsar-bookie + nc -z localhost 6650 + i=1 + '[' 1 -gt 20 ']' + sleep 2 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.resolve_lock.cli.2929.out cli tso query --pd=http://127.0.0.1:2379 wait process 6161 exit for 10-th time... Verifying downstream PD is started... check diff failed 5-th time, retry later /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (6161) - No such process wait process 6161 exit for 11-th time... process 6161 already exit [Fri Apr 4 15:34:04 CST 2025] <<<<<< START cdc server in ddl_manager case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ExecuteDDLSlowly=return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_manager.67956797.out server --log-file /tmp/tidb_cdc_test/ddl_manager/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_manager/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + nc -z localhost 6650 + i=2 + '[' 2 -gt 20 ']' + sleep 2 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8301 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 07:34:04 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [817 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e961eba9-a1b2-4f89-8df7-d4d4e3c56689 {"id":"e961eba9-a1b2-4f89-8df7-d4d4e3c56689","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752042} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffb871bccf e961eba9-a1b2-4f89-8df7-d4d4e3c56689 /tidb/cdc/default/default/upstream/7489357962550911928 {"id":7489357962550911928,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e961eba9-a1b2-4f89-8df7-d4d4e3c56689 {"id":"e961eba9-a1b2-4f89-8df7-d4d4e3c56689","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752042} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffb871bccf e961eba9-a1b2-4f89-8df7-d4d4e3c56689 /tidb/cdc/default/default/upstream/7489357962550911928 {"id":7489357962550911928,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e961eba9-a1b2-4f89-8df7-d4d4e3c56689 {"id":"e961eba9-a1b2-4f89-8df7-d4d4e3c56689","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752042} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffb871bccf e961eba9-a1b2-4f89-8df7-d4d4e3c56689 /tidb/cdc/default/default/upstream/7489357962550911928 {"id":7489357962550911928,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x ++ date + echo '[Fri Apr 4 15:34:04 CST 2025] <<<<<< START pulsar cluster in processor_resolved_ts_fallback case >>>>>>' [Fri Apr 4 15:34:04 CST 2025] <<<<<< START pulsar cluster in processor_resolved_ts_fallback case >>>>>> + workdir=/tmp/tidb_cdc_test/processor_resolved_ts_fallback + cluster_type=normal + cd /tmp/tidb_cdc_test/processor_resolved_ts_fallback + DEFAULT_PULSAR_HOME=/usr/local/pulsar + pulsar_dir=/usr/local/pulsar ++ cat + mtls_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderTls brokerClientTlsEnabled=true brokerClientTrustCertsFilePath=/tmp/tidb_cdc_test/processor_resolved_ts_fallback/ca.cert.pem brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls brokerClientAuthenticationParameters={"tlsCertFile":"/tmp/tidb_cdc_test/processor_resolved_ts_fallback/broker_client.cert.pem","tlsKeyFile":"/tmp/tidb_cdc_test/processor_resolved_ts_fallback/broker_client.key-pk8.pem"} brokerServicePortTls=6651 webServicePortTls=8443 tlsTrustCertsFilePath=/tmp/tidb_cdc_test/processor_resolved_ts_fallback/ca.cert.pem tlsCertificateFilePath=/tmp/tidb_cdc_test/processor_resolved_ts_fallback/server.cert.pem tlsKeyFilePath=/tmp/tidb_cdc_test/processor_resolved_ts_fallback/server.key-pk8.pem tlsRequireTrustedClientCertOnConnect=true tlsAllowInsecureConnection=false tlsCertRefreshCheckDurationSec=300' ++ cat + normal_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' ++ cat + mtls_client_conf=' webServiceUrl=https://localhost:8443/ brokerServiceUrl=pulsar+ssl://localhost:6651/ authPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls authParams=tlsCertFile:/tmp/tidb_cdc_test/processor_resolved_ts_fallback/broker_client.cert.pem,tlsKeyFile:/tmp/tidb_cdc_test/processor_resolved_ts_fallback/broker_client.key-pk8.pem tlsTrustCertsFilePath=/tmp/tidb_cdc_test/processor_resolved_ts_fallback/ca.cert.pem' ++ cat + oauth_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/ authPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 authParams={"privateKey":"/tmp/tidb_cdc_test/processor_resolved_ts_fallback/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"}' ++ cat + oauth_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderToken brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 brokerClientAuthenticationParameters={"privateKey":"file:///tmp/tidb_cdc_test/processor_resolved_ts_fallback/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"} tokenSecretKey=data:;base64,U0poWDM2X0thcFlTeWJCdEpxMzVseFhfQnJyNExSVVJTa203UW1YSkdteThwVUZXOUVJT2NWUVBzeWt6OS1qag==' ++ cat + credential_json=' { "client_id":"1234", "client_secret":"e0KVlA2EiBfjoN13olyZd2kv1KL", "audience":"cdc-api-uri", "issuer_url":"http://localhost:9096", "type": "client_credentials" }' ++ cat + cert_server_conf='[ req ] default_bits = 2048 prompt = no default_md = sha256 distinguished_name = dn [ v3_ext ] authorityKeyIdentifier=keyid,issuer:always basicConstraints=CA:FALSE keyUsage=critical, digitalSignature, keyEncipherment extendedKeyUsage=serverAuth subjectAltName=@alt_names [ dn ] CN = server [ alt_names ] DNS.1 = localhost IP.1 = 127.0.0.1' + echo ' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' + cp /usr/local/pulsar/conf/standalone.conf /tmp/tidb_cdc_test/processor_resolved_ts_fallback/pulsar_standalone.conf + pulsar_port=6650 + '[' normal == mtls ']' + '[' normal == oauth ']' + echo 'no cluster type specified, using default configuration.' no cluster type specified, using default configuration. ++ date + echo '[Fri Apr 4 15:34:04 CST 2025] <<<<<< START pulsar cluster in normal mode in processor_resolved_ts_fallback case >>>>>>' [Fri Apr 4 15:34:04 CST 2025] <<<<<< START pulsar cluster in normal mode in processor_resolved_ts_fallback case >>>>>> + echo 'Waiting for pulsar port to be ready...' Waiting for pulsar port to be ready... + i=0 + /usr/local/pulsar/bin/pulsar standalone --config /tmp/tidb_cdc_test/processor_resolved_ts_fallback/pulsar_standalone.conf -nfw --metadata-dir /tmp/tidb_cdc_test/processor_resolved_ts_fallback/pulsar-metadata --bookkeeper-dir /tmp/tidb_cdc_test/processor_resolved_ts_fallback/pulsar-bookie + nc -z localhost 6650 + i=1 + '[' 1 -gt 20 ']' + sleep 2 [Fri Apr 4 15:34:05 CST 2025] <<<<<< START cdc server in batch_add_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.batch_add_table.29622964.out server --log-file /tmp/tidb_cdc_test/batch_add_table/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/batch_add_table/cdc_data --cluster-id default + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + nc -z localhost 6650 + i=2 + '[' 2 -gt 20 ']' + sleep 2 + nc -z localhost 6650 + i=2 + '[' 2 -gt 20 ']' + sleep 2 Verifying downstream PD is started... + set +x + tso='457114135811325953 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 457114135811325953 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 4 15:34:05 CST 2025] <<<<<< START cdc server in resolve_lock case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.resolve_lock.29692971.out server --log-file /tmp/tidb_cdc_test/resolve_lock/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/resolve_lock/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + nc -z localhost 6650 + echo 'Waiting for pulsar namespace to be ready...' Waiting for pulsar namespace to be ready... + i=0 + /usr/local/pulsar/bin/pulsar-admin namespaces list public table split_region.test1 exists table split_region.test2 not exists for 1-th check, retry later check diff failed 6-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 07:34:05 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [817 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9b08204c-1d01-4324-aa9c-79d200ddd772 {"id":"9b08204c-1d01-4324-aa9c-79d200ddd772","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752043} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffb86f66d6 9b08204c-1d01-4324-aa9c-79d200ddd772 /tidb/cdc/default/default/upstream/7489357955358834369 {"id":7489357955358834369,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9b08204c-1d01-4324-aa9c-79d200ddd772 {"id":"9b08204c-1d01-4324-aa9c-79d200ddd772","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752043} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffb86f66d6 9b08204c-1d01-4324-aa9c-79d200ddd772 /tidb/cdc/default/default/upstream/7489357955358834369 {"id":7489357955358834369,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9b08204c-1d01-4324-aa9c-79d200ddd772 {"id":"9b08204c-1d01-4324-aa9c-79d200ddd772","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752043} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffb86f66d6 9b08204c-1d01-4324-aa9c-79d200ddd772 /tidb/cdc/default/default/upstream/7489357955358834369 {"id":7489357955358834369,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x ++ date + echo '[Fri Apr 4 15:34:06 CST 2025] <<<<<< START pulsar cluster in tiflash case >>>>>>' [Fri Apr 4 15:34:06 CST 2025] <<<<<< START pulsar cluster in tiflash case >>>>>> + workdir=/tmp/tidb_cdc_test/tiflash + cluster_type=mtls + cd /tmp/tidb_cdc_test/tiflash + DEFAULT_PULSAR_HOME=/usr/local/pulsar + pulsar_dir=/usr/local/pulsar ++ cat + mtls_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderTls brokerClientTlsEnabled=true brokerClientTrustCertsFilePath=/tmp/tidb_cdc_test/tiflash/ca.cert.pem brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls brokerClientAuthenticationParameters={"tlsCertFile":"/tmp/tidb_cdc_test/tiflash/broker_client.cert.pem","tlsKeyFile":"/tmp/tidb_cdc_test/tiflash/broker_client.key-pk8.pem"} brokerServicePortTls=6651 webServicePortTls=8443 tlsTrustCertsFilePath=/tmp/tidb_cdc_test/tiflash/ca.cert.pem tlsCertificateFilePath=/tmp/tidb_cdc_test/tiflash/server.cert.pem tlsKeyFilePath=/tmp/tidb_cdc_test/tiflash/server.key-pk8.pem tlsRequireTrustedClientCertOnConnect=true tlsAllowInsecureConnection=false tlsCertRefreshCheckDurationSec=300' ++ cat + normal_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' ++ cat + mtls_client_conf=' webServiceUrl=https://localhost:8443/ brokerServiceUrl=pulsar+ssl://localhost:6651/ authPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls authParams=tlsCertFile:/tmp/tidb_cdc_test/tiflash/broker_client.cert.pem,tlsKeyFile:/tmp/tidb_cdc_test/tiflash/broker_client.key-pk8.pem tlsTrustCertsFilePath=/tmp/tidb_cdc_test/tiflash/ca.cert.pem' ++ cat + oauth_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/ authPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 authParams={"privateKey":"/tmp/tidb_cdc_test/tiflash/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"}' ++ cat + oauth_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderToken brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 brokerClientAuthenticationParameters={"privateKey":"file:///tmp/tidb_cdc_test/tiflash/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"} tokenSecretKey=data:;base64,U0poWDM2X0thcFlTeWJCdEpxMzVseFhfQnJyNExSVVJTa203UW1YSkdteThwVUZXOUVJT2NWUVBzeWt6OS1qag==' ++ cat + credential_json=' { "client_id":"1234", "client_secret":"e0KVlA2EiBfjoN13olyZd2kv1KL", "audience":"cdc-api-uri", "issuer_url":"http://localhost:9096", "type": "client_credentials" }' ++ cat + cert_server_conf='[ req ] default_bits = 2048 prompt = no default_md = sha256 distinguished_name = dn [ v3_ext ] authorityKeyIdentifier=keyid,issuer:always basicConstraints=CA:FALSE keyUsage=critical, digitalSignature, keyEncipherment extendedKeyUsage=serverAuth subjectAltName=@alt_names [ dn ] CN = server [ alt_names ] DNS.1 = localhost IP.1 = 127.0.0.1' + echo ' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' + cp /usr/local/pulsar/conf/standalone.conf /tmp/tidb_cdc_test/tiflash/pulsar_standalone.conf + pulsar_port=6650 + '[' mtls == mtls ']' + pulsar_port=6651 + gen_mtls_config + openssl genrsa -out ca.key.pem 2048 Generating RSA private key, 2048 bit long modulus (2 primes) .......+++++ .....+++++ e is 65537 (0x010001) + openssl req -x509 -new -nodes -key ca.key.pem -subj /CN=CARoot -days 365 -out ca.cert.pem + openssl genrsa -out server.key.pem 2048 Generating RSA private key, 2048 bit long modulus (2 primes) .........................+++++ ..................................................+++++ e is 65537 (0x010001) + openssl pkcs8 -topk8 -inform PEM -outform PEM -in server.key.pem -out server.key-pk8.pem -nocrypt + echo '[ req ] default_bits = 2048 prompt = no default_md = sha256 distinguished_name = dn [ v3_ext ] authorityKeyIdentifier=keyid,issuer:always basicConstraints=CA:FALSE keyUsage=critical, digitalSignature, keyEncipherment extendedKeyUsage=serverAuth subjectAltName=@alt_names [ dn ] CN = server [ alt_names ] DNS.1 = localhost IP.1 = 127.0.0.1' + openssl req -new -config server.conf -key server.key.pem -out server.csr.pem -sha256 + openssl x509 -req -in server.csr.pem -CA ca.cert.pem -CAkey ca.key.pem -CAcreateserial -out server.cert.pem -days 365 -extensions v3_ext -extfile server.conf -sha256 Signature ok subject=CN = server Getting CA Private Key + openssl genrsa -out broker_client.key.pem 2048 Generating RSA private key, 2048 bit long modulus (2 primes) ....+++++ ................+++++ e is 65537 (0x010001) + openssl pkcs8 -topk8 -inform PEM -outform PEM -in broker_client.key.pem -out broker_client.key-pk8.pem -nocrypt + openssl req -new -subj /CN=broker_client -key broker_client.key.pem -out broker_client.csr.pem -sha256 + openssl x509 -req -in broker_client.csr.pem -CA ca.cert.pem -CAkey ca.key.pem -CAcreateserial -out broker_client.cert.pem -days 365 -sha256 Signature ok subject=CN = broker_client Getting CA Private Key + echo ' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderTls brokerClientTlsEnabled=true brokerClientTrustCertsFilePath=/tmp/tidb_cdc_test/tiflash/ca.cert.pem brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls brokerClientAuthenticationParameters={"tlsCertFile":"/tmp/tidb_cdc_test/tiflash/broker_client.cert.pem","tlsKeyFile":"/tmp/tidb_cdc_test/tiflash/broker_client.key-pk8.pem"} brokerServicePortTls=6651 webServicePortTls=8443 tlsTrustCertsFilePath=/tmp/tidb_cdc_test/tiflash/ca.cert.pem tlsCertificateFilePath=/tmp/tidb_cdc_test/tiflash/server.cert.pem tlsKeyFilePath=/tmp/tidb_cdc_test/tiflash/server.key-pk8.pem tlsRequireTrustedClientCertOnConnect=true tlsAllowInsecureConnection=false tlsCertRefreshCheckDurationSec=300' + echo ' webServiceUrl=https://localhost:8443/ brokerServiceUrl=pulsar+ssl://localhost:6651/ authPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls authParams=tlsCertFile:/tmp/tidb_cdc_test/tiflash/broker_client.cert.pem,tlsKeyFile:/tmp/tidb_cdc_test/tiflash/broker_client.key-pk8.pem tlsTrustCertsFilePath=/tmp/tidb_cdc_test/tiflash/ca.cert.pem' ++ date + echo '[Fri Apr 4 15:34:06 CST 2025] <<<<<< START pulsar cluster in mtls mode in tiflash case >>>>>>' [Fri Apr 4 15:34:06 CST 2025] <<<<<< START pulsar cluster in mtls mode in tiflash case >>>>>> + echo 'Waiting for pulsar port to be ready...' Waiting for pulsar port to be ready... + i=0 + nc -z localhost 6651 + /usr/local/pulsar/bin/pulsar standalone --config /tmp/tidb_cdc_test/tiflash/pulsar_standalone.conf -nfw --metadata-dir /tmp/tidb_cdc_test/tiflash/pulsar-metadata --bookkeeper-dir /tmp/tidb_cdc_test/tiflash/pulsar-bookie + i=1 + '[' 1 -gt 20 ']' + sleep 2 Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 07:34:06 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [817 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d3cd69a0-b95b-40d3-9e20-fdd8efbfb415 {"id":"d3cd69a0-b95b-40d3-9e20-fdd8efbfb415","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752043} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffb86d79ec d3cd69a0-b95b-40d3-9e20-fdd8efbfb415 /tidb/cdc/default/default/upstream/7489357950984869366 {"id":7489357950984869366,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d3cd69a0-b95b-40d3-9e20-fdd8efbfb415 {"id":"d3cd69a0-b95b-40d3-9e20-fdd8efbfb415","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752043} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffb86d79ec d3cd69a0-b95b-40d3-9e20-fdd8efbfb415 /tidb/cdc/default/default/upstream/7489357950984869366 {"id":7489357950984869366,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d3cd69a0-b95b-40d3-9e20-fdd8efbfb415 {"id":"d3cd69a0-b95b-40d3-9e20-fdd8efbfb415","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752043} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffb86d79ec d3cd69a0-b95b-40d3-9e20-fdd8efbfb415 /tidb/cdc/default/default/upstream/7489357950984869366 {"id":7489357950984869366,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x ++ date + echo '[Fri Apr 4 15:34:06 CST 2025] <<<<<< START pulsar cluster in simple case >>>>>>' [Fri Apr 4 15:34:06 CST 2025] <<<<<< START pulsar cluster in simple case >>>>>> + workdir=/tmp/tidb_cdc_test/simple + cluster_type=normal + cd /tmp/tidb_cdc_test/simple + DEFAULT_PULSAR_HOME=/usr/local/pulsar + pulsar_dir=/usr/local/pulsar ++ cat + mtls_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderTls brokerClientTlsEnabled=true brokerClientTrustCertsFilePath=/tmp/tidb_cdc_test/simple/ca.cert.pem brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls brokerClientAuthenticationParameters={"tlsCertFile":"/tmp/tidb_cdc_test/simple/broker_client.cert.pem","tlsKeyFile":"/tmp/tidb_cdc_test/simple/broker_client.key-pk8.pem"} brokerServicePortTls=6651 webServicePortTls=8443 tlsTrustCertsFilePath=/tmp/tidb_cdc_test/simple/ca.cert.pem tlsCertificateFilePath=/tmp/tidb_cdc_test/simple/server.cert.pem tlsKeyFilePath=/tmp/tidb_cdc_test/simple/server.key-pk8.pem tlsRequireTrustedClientCertOnConnect=true tlsAllowInsecureConnection=false tlsCertRefreshCheckDurationSec=300' ++ cat + normal_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' ++ cat + mtls_client_conf=' webServiceUrl=https://localhost:8443/ brokerServiceUrl=pulsar+ssl://localhost:6651/ authPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls authParams=tlsCertFile:/tmp/tidb_cdc_test/simple/broker_client.cert.pem,tlsKeyFile:/tmp/tidb_cdc_test/simple/broker_client.key-pk8.pem tlsTrustCertsFilePath=/tmp/tidb_cdc_test/simple/ca.cert.pem' ++ cat + oauth_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/ authPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 authParams={"privateKey":"/tmp/tidb_cdc_test/simple/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"}' ++ cat + oauth_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderToken brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 brokerClientAuthenticationParameters={"privateKey":"file:///tmp/tidb_cdc_test/simple/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"} tokenSecretKey=data:;base64,U0poWDM2X0thcFlTeWJCdEpxMzVseFhfQnJyNExSVVJTa203UW1YSkdteThwVUZXOUVJT2NWUVBzeWt6OS1qag==' ++ cat + credential_json=' { "client_id":"1234", "client_secret":"e0KVlA2EiBfjoN13olyZd2kv1KL", "audience":"cdc-api-uri", "issuer_url":"http://localhost:9096", "type": "client_credentials" }' ++ cat + cert_server_conf='[ req ] default_bits = 2048 prompt = no default_md = sha256 distinguished_name = dn [ v3_ext ] authorityKeyIdentifier=keyid,issuer:always basicConstraints=CA:FALSE keyUsage=critical, digitalSignature, keyEncipherment extendedKeyUsage=serverAuth subjectAltName=@alt_names [ dn ] CN = server [ alt_names ] DNS.1 = localhost IP.1 = 127.0.0.1' + echo ' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' + cp /usr/local/pulsar/conf/standalone.conf /tmp/tidb_cdc_test/simple/pulsar_standalone.conf + pulsar_port=6650 + '[' normal == mtls ']' + '[' normal == oauth ']' + echo 'no cluster type specified, using default configuration.' no cluster type specified, using default configuration. ++ date + echo '[Fri Apr 4 15:34:06 CST 2025] <<<<<< START pulsar cluster in normal mode in simple case >>>>>>' [Fri Apr 4 15:34:06 CST 2025] <<<<<< START pulsar cluster in normal mode in simple case >>>>>> + echo 'Waiting for pulsar port to be ready...' Waiting for pulsar port to be ready... + i=0 + nc -z localhost 6650 + /usr/local/pulsar/bin/pulsar standalone --config /tmp/tidb_cdc_test/simple/pulsar_standalone.conf -nfw --metadata-dir /tmp/tidb_cdc_test/simple/pulsar-metadata --bookkeeper-dir /tmp/tidb_cdc_test/simple/pulsar-bookie + i=1 + '[' 1 -gt 20 ']' + sleep 2 Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + nc -z localhost 6650 + echo 'Waiting for pulsar namespace to be ready...' Waiting for pulsar namespace to be ready... + i=0 + /usr/local/pulsar/bin/pulsar-admin namespaces list public + nc -z localhost 6650 + i=2 + '[' 2 -gt 20 ']' + sleep 2 + nc -z localhost 6650 + echo 'Waiting for pulsar namespace to be ready...' Waiting for pulsar namespace to be ready... + i=0 + /usr/local/pulsar/bin/pulsar-admin namespaces list public + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > public/default ++ date + echo '[Fri Apr 4 15:34:07 CST 2025] <<<<<< pulsar is ready >>>>>>' [Fri Apr 4 15:34:07 CST 2025] <<<<<< pulsar is ready >>>>>> + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.canal_json_content_compatible.cli.8065.out cli changefeed create '--sink-uri=pulsar://127.0.0.1:6650/ticdc-canal-json-content-compatible?protocol=canal-json&enable-tidb-extension=true' table split_region.test2 exists check diff failed 1-th time, retry later + nc -z localhost 6650 + echo 'Waiting for pulsar namespace to be ready...' Waiting for pulsar namespace to be ready... + i=0 + /usr/local/pulsar/bin/pulsar-admin namespaces list public check diff successfully + nc -z localhost 6651 + i=2 + '[' 2 -gt 20 ']' + sleep 2 Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 07:34:08 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3971 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/ddl-manager {UpstreamID:7489357833116153411 Namespace:default ID:ddl-manager SinkURI:pulsar://127.0.0.1:6650/ticdc-ddl-mamager-test-6367?protocol=canal-json&enable-tidb-extension=true CreateTime:2025-04-04 15:33:42.333112167 +0800 CST StartTs:457114129989369859 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0018d4f30 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-10-ga306f4894 Epoch:457114130133549060} {CheckpointTs:457114130919981071 MinTableBarrierTs:457114130919981071 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7caeff6d-5142-4d33-b057-597856bd1ae2 {"id":"7caeff6d-5142-4d33-b057-597856bd1ae2","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752044} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffb7ffa74d 7caeff6d-5142-4d33-b057-597856bd1ae2 /tidb/cdc/default/default/changefeed/info/ddl-manager {"upstream-id":7489357833116153411,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"pulsar://127.0.0.1:6650/ticdc-ddl-mamager-test-6367?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2025-04-04T15:33:42.333112167+08:00","start-ts":457114129989369859,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"pulsar-config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-10-ga306f4894","epoch":457114130133549060} /tidb/cdc/default/default/changefeed/status/ddl-manager {"checkpoint-ts":457114130919981071,"min-table-barrier-ts":457114130919981071,"admin-job-type":0} /tidb/cdc/default/default/task/position/7caeff6d-5142-4d33-b057-597856bd1ae2/ddl-manager {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489357833116153411 {"id":7489357833116153411,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-manager {UpstreamID:7489357833116153411 Namespace:default ID:ddl-manager SinkURI:pulsar://127.0.0.1:6650/ticdc-ddl-mamager-test-6367?protocol=canal-json&enable-tidb-extension=true CreateTime:2025-04-04 15:33:42.333112167 +0800 CST StartTs:457114129989369859 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0018d4f30 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-10-ga306f4894 Epoch:457114130133549060} {CheckpointTs:457114130919981071 MinTableBarrierTs:457114130919981071 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7caeff6d-5142-4d33-b057-597856bd1ae2 {"id":"7caeff6d-5142-4d33-b057-597856bd1ae2","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752044} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffb7ffa74d 7caeff6d-5142-4d33-b057-597856bd1ae2 /tidb/cdc/default/default/changefeed/info/ddl-manager {"upstream-id":7489357833116153411,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"pulsar://127.0.0.1:6650/ticdc-ddl-mamager-test-6367?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2025-04-04T15:33:42.333112167+08:00","start-ts":457114129989369859,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"pulsar-config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-10-ga306f4894","epoch":457114130133549060} /tidb/cdc/default/default/changefeed/status/ddl-manager {"checkpoint-ts":457114130919981071,"min-table-barrier-ts":457114130919981071,"admin-job-type":0} /tidb/cdc/default/default/task/position/7caeff6d-5142-4d33-b057-597856bd1ae2/ddl-manager {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489357833116153411 {"id":7489357833116153411,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-manager {UpstreamID:7489357833116153411 Namespace:default ID:ddl-manager SinkURI:pulsar://127.0.0.1:6650/ticdc-ddl-mamager-test-6367?protocol=canal-json&enable-tidb-extension=true CreateTime:2025-04-04 15:33:42.333112167 +0800 CST StartTs:457114129989369859 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0018d4f30 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-10-ga306f4894 Epoch:457114130133549060} {CheckpointTs:457114130919981071 MinTableBarrierTs:457114130919981071 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7caeff6d-5142-4d33-b057-597856bd1ae2 {"id":"7caeff6d-5142-4d33-b057-597856bd1ae2","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752044} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffb7ffa74d 7caeff6d-5142-4d33-b057-597856bd1ae2 /tidb/cdc/default/default/changefeed/info/ddl-manager {"upstream-id":7489357833116153411,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"pulsar://127.0.0.1:6650/ticdc-ddl-mamager-test-6367?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2025-04-04T15:33:42.333112167+08:00","start-ts":457114129989369859,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"pulsar-config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-10-ga306f4894","epoch":457114130133549060} /tidb/cdc/default/default/changefeed/status/ddl-manager {"checkpoint-ts":457114130919981071,"min-table-barrier-ts":457114130919981071,"admin-job-type":0} /tidb/cdc/default/default/task/position/7caeff6d-5142-4d33-b057-597856bd1ae2/ddl-manager {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489357833116153411 {"id":7489357833116153411,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Fri Apr 4 15:34:08 CST 2025] <<<<<< START cdc server in ddl_manager case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ExecuteDDLSlowly=return(true)' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_manager.68656867.out server --log-file /tmp/tidb_cdc_test/ddl_manager/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_manager/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > Create changefeed successfully! ID: 73844e2b-1528-481b-82f3-05204f491dce Info: {"upstream_id":7489357935344335316,"namespace":"default","id":"73844e2b-1528-481b-82f3-05204f491dce","sink_uri":"pulsar://127.0.0.1:6650/ticdc-canal-json-content-compatible?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2025-04-04T15:34:08.458729929+08:00","start_ts":457114136837357569,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"pulsar_config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-10-ga306f4894","resolved_ts":457114136837357569,"checkpoint_ts":457114136837357569,"checkpoint_time":"2025-04-04 15:34:07.872"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 07:34:08 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [817 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/52bef7e8-0ae4-4a19-b031-58a76bf16a94 {"id":"52bef7e8-0ae4-4a19-b031-58a76bf16a94","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752045} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffb8793dd4 52bef7e8-0ae4-4a19-b031-58a76bf16a94 /tidb/cdc/default/default/upstream/7489357967512829777 {"id":7489357967512829777,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/52bef7e8-0ae4-4a19-b031-58a76bf16a94 {"id":"52bef7e8-0ae4-4a19-b031-58a76bf16a94","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752045} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffb8793dd4 52bef7e8-0ae4-4a19-b031-58a76bf16a94 /tidb/cdc/default/default/upstream/7489357967512829777 {"id":7489357967512829777,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/52bef7e8-0ae4-4a19-b031-58a76bf16a94 {"id":"52bef7e8-0ae4-4a19-b031-58a76bf16a94","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752045} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffb8793dd4 52bef7e8-0ae4-4a19-b031-58a76bf16a94 /tidb/cdc/default/default/upstream/7489357967512829777 {"id":7489357967512829777,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x ++ date + echo '[Fri Apr 4 15:34:08 CST 2025] <<<<<< START pulsar cluster in batch_add_table case >>>>>>' [Fri Apr 4 15:34:08 CST 2025] <<<<<< START pulsar cluster in batch_add_table case >>>>>> + workdir=/tmp/tidb_cdc_test/batch_add_table + cluster_type=normal + cd /tmp/tidb_cdc_test/batch_add_table + DEFAULT_PULSAR_HOME=/usr/local/pulsar + pulsar_dir=/usr/local/pulsar ++ cat + mtls_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderTls brokerClientTlsEnabled=true brokerClientTrustCertsFilePath=/tmp/tidb_cdc_test/batch_add_table/ca.cert.pem brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls brokerClientAuthenticationParameters={"tlsCertFile":"/tmp/tidb_cdc_test/batch_add_table/broker_client.cert.pem","tlsKeyFile":"/tmp/tidb_cdc_test/batch_add_table/broker_client.key-pk8.pem"} brokerServicePortTls=6651 webServicePortTls=8443 tlsTrustCertsFilePath=/tmp/tidb_cdc_test/batch_add_table/ca.cert.pem tlsCertificateFilePath=/tmp/tidb_cdc_test/batch_add_table/server.cert.pem tlsKeyFilePath=/tmp/tidb_cdc_test/batch_add_table/server.key-pk8.pem tlsRequireTrustedClientCertOnConnect=true tlsAllowInsecureConnection=false tlsCertRefreshCheckDurationSec=300' ++ cat + normal_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' ++ cat + mtls_client_conf=' webServiceUrl=https://localhost:8443/ brokerServiceUrl=pulsar+ssl://localhost:6651/ authPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls authParams=tlsCertFile:/tmp/tidb_cdc_test/batch_add_table/broker_client.cert.pem,tlsKeyFile:/tmp/tidb_cdc_test/batch_add_table/broker_client.key-pk8.pem tlsTrustCertsFilePath=/tmp/tidb_cdc_test/batch_add_table/ca.cert.pem' ++ cat + oauth_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/ authPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 authParams={"privateKey":"/tmp/tidb_cdc_test/batch_add_table/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"}' ++ cat + oauth_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderToken brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 brokerClientAuthenticationParameters={"privateKey":"file:///tmp/tidb_cdc_test/batch_add_table/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"} tokenSecretKey=data:;base64,U0poWDM2X0thcFlTeWJCdEpxMzVseFhfQnJyNExSVVJTa203UW1YSkdteThwVUZXOUVJT2NWUVBzeWt6OS1qag==' ++ cat + credential_json=' { "client_id":"1234", "client_secret":"e0KVlA2EiBfjoN13olyZd2kv1KL", "audience":"cdc-api-uri", "issuer_url":"http://localhost:9096", "type": "client_credentials" }' ++ cat + cert_server_conf='[ req ] default_bits = 2048 prompt = no default_md = sha256 distinguished_name = dn [ v3_ext ] authorityKeyIdentifier=keyid,issuer:always basicConstraints=CA:FALSE keyUsage=critical, digitalSignature, keyEncipherment extendedKeyUsage=serverAuth subjectAltName=@alt_names [ dn ] CN = server [ alt_names ] DNS.1 = localhost IP.1 = 127.0.0.1' + echo ' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' + cp /usr/local/pulsar/conf/standalone.conf /tmp/tidb_cdc_test/batch_add_table/pulsar_standalone.conf + pulsar_port=6650 + '[' normal == mtls ']' + '[' normal == oauth ']' + echo 'no cluster type specified, using default configuration.' no cluster type specified, using default configuration. ++ date + echo '[Fri Apr 4 15:34:08 CST 2025] <<<<<< START pulsar cluster in normal mode in batch_add_table case >>>>>>' [Fri Apr 4 15:34:08 CST 2025] <<<<<< START pulsar cluster in normal mode in batch_add_table case >>>>>> + echo 'Waiting for pulsar port to be ready...' Waiting for pulsar port to be ready... + i=0 + nc -z localhost 6650 + /usr/local/pulsar/bin/pulsar standalone --config /tmp/tidb_cdc_test/batch_add_table/pulsar_standalone.conf -nfw --metadata-dir /tmp/tidb_cdc_test/batch_add_table/pulsar-metadata --bookkeeper-dir /tmp/tidb_cdc_test/batch_add_table/pulsar-bookie + i=1 + '[' 1 -gt 20 ']' + sleep 2 Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + nc -z localhost 6650 + i=2 + '[' 2 -gt 20 ']' + sleep 2 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489357992790424245 TiDB Cluster ID. tikv_gc_leader_uuid 657fee2b538000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-9bld-w6hq9, pid:8490, start at 2025-04-04 15:34:07.995387785 +0800 CST m=+1.529623861 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:36:08.005 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:34:08.001 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:24:08.001 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489357992790424245 TiDB Cluster ID. tikv_gc_leader_uuid 657fee2b538000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-9bld-w6hq9, pid:8490, start at 2025-04-04 15:34:07.995387785 +0800 CST m=+1.529623861 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:36:08.005 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:34:08.001 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:24:08.001 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489357998292053260 TiDB Cluster ID. tikv_gc_leader_uuid 657fee2b5f4000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-9bld-w6hq9, pid:8554, start at 2025-04-04 15:34:08.014111746 +0800 CST m=+1.485716027 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:36:08.021 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:34:07.997 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:24:07.997 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... public/default + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 07:34:08 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [817 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/633e9064-d196-49f6-8905-589cdfad7342 {"id":"633e9064-d196-49f6-8905-589cdfad7342","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752045} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffb87b1dd5 633e9064-d196-49f6-8905-589cdfad7342 /tidb/cdc/default/default/upstream/7489357969709569958 {"id":7489357969709569958,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/633e9064-d196-49f6-8905-589cdfad7342 {"id":"633e9064-d196-49f6-8905-589cdfad7342","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752045} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffb87b1dd5 633e9064-d196-49f6-8905-589cdfad7342 /tidb/cdc/default/default/upstream/7489357969709569958 {"id":7489357969709569958,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/633e9064-d196-49f6-8905-589cdfad7342 {"id":"633e9064-d196-49f6-8905-589cdfad7342","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752045} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffb87b1dd5 633e9064-d196-49f6-8905-589cdfad7342 /tidb/cdc/default/default/upstream/7489357969709569958 {"id":7489357969709569958,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x ++ date + echo '[Fri Apr 4 15:34:08 CST 2025] <<<<<< START pulsar cluster in resolve_lock case >>>>>>' [Fri Apr 4 15:34:08 CST 2025] <<<<<< START pulsar cluster in resolve_lock case >>>>>> + workdir=/tmp/tidb_cdc_test/resolve_lock + cluster_type=normal + cd /tmp/tidb_cdc_test/resolve_lock + DEFAULT_PULSAR_HOME=/usr/local/pulsar + pulsar_dir=/usr/local/pulsar ++ cat + mtls_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderTls brokerClientTlsEnabled=true brokerClientTrustCertsFilePath=/tmp/tidb_cdc_test/resolve_lock/ca.cert.pem brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls brokerClientAuthenticationParameters={"tlsCertFile":"/tmp/tidb_cdc_test/resolve_lock/broker_client.cert.pem","tlsKeyFile":"/tmp/tidb_cdc_test/resolve_lock/broker_client.key-pk8.pem"} brokerServicePortTls=6651 webServicePortTls=8443 tlsTrustCertsFilePath=/tmp/tidb_cdc_test/resolve_lock/ca.cert.pem tlsCertificateFilePath=/tmp/tidb_cdc_test/resolve_lock/server.cert.pem tlsKeyFilePath=/tmp/tidb_cdc_test/resolve_lock/server.key-pk8.pem tlsRequireTrustedClientCertOnConnect=true tlsAllowInsecureConnection=false tlsCertRefreshCheckDurationSec=300' ++ cat + normal_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' ++ cat + mtls_client_conf=' webServiceUrl=https://localhost:8443/ brokerServiceUrl=pulsar+ssl://localhost:6651/ authPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls authParams=tlsCertFile:/tmp/tidb_cdc_test/resolve_lock/broker_client.cert.pem,tlsKeyFile:/tmp/tidb_cdc_test/resolve_lock/broker_client.key-pk8.pem tlsTrustCertsFilePath=/tmp/tidb_cdc_test/resolve_lock/ca.cert.pem' ++ cat + oauth_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/ authPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 authParams={"privateKey":"/tmp/tidb_cdc_test/resolve_lock/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"}' ++ cat + oauth_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderToken brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 brokerClientAuthenticationParameters={"privateKey":"file:///tmp/tidb_cdc_test/resolve_lock/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"} tokenSecretKey=data:;base64,U0poWDM2X0thcFlTeWJCdEpxMzVseFhfQnJyNExSVVJTa203UW1YSkdteThwVUZXOUVJT2NWUVBzeWt6OS1qag==' ++ cat + credential_json=' { "client_id":"1234", "client_secret":"e0KVlA2EiBfjoN13olyZd2kv1KL", "audience":"cdc-api-uri", "issuer_url":"http://localhost:9096", "type": "client_credentials" }' ++ cat + cert_server_conf='[ req ] default_bits = 2048 prompt = no default_md = sha256 distinguished_name = dn [ v3_ext ] authorityKeyIdentifier=keyid,issuer:always basicConstraints=CA:FALSE keyUsage=critical, digitalSignature, keyEncipherment extendedKeyUsage=serverAuth subjectAltName=@alt_names [ dn ] CN = server [ alt_names ] DNS.1 = localhost IP.1 = 127.0.0.1' + echo ' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' + cp /usr/local/pulsar/conf/standalone.conf /tmp/tidb_cdc_test/resolve_lock/pulsar_standalone.conf + pulsar_port=6650 + '[' normal == mtls ']' + '[' normal == oauth ']' + echo 'no cluster type specified, using default configuration.' no cluster type specified, using default configuration. ++ date + echo '[Fri Apr 4 15:34:08 CST 2025] <<<<<< START pulsar cluster in normal mode in resolve_lock case >>>>>>' [Fri Apr 4 15:34:08 CST 2025] <<<<<< START pulsar cluster in normal mode in resolve_lock case >>>>>> + echo 'Waiting for pulsar port to be ready...' Waiting for pulsar port to be ready... + i=0 + nc -z localhost 6650 + /usr/local/pulsar/bin/pulsar standalone --config /tmp/tidb_cdc_test/resolve_lock/pulsar_standalone.conf -nfw --metadata-dir /tmp/tidb_cdc_test/resolve_lock/pulsar-metadata --bookkeeper-dir /tmp/tidb_cdc_test/resolve_lock/pulsar-bookie + i=1 + '[' 1 -gt 20 ']' + sleep 2 Logging trace to /tmp/tidb_cdc_test/multi_capture/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/multi_capture/tiflash/log/error.log arg matches is ArgMatches { args: {"addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/multi_capture/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/multi_capture/tiflash/db/proxy"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/multi_capture/tiflash/log/proxy.log"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } ++ date + echo '[Fri Apr 4 15:34:08 CST 2025] <<<<<< pulsar is ready >>>>>>' [Fri Apr 4 15:34:08 CST 2025] <<<<<< pulsar is ready >>>>>> public/default check diff failed 1-th time, retry later + nc -z localhost 6650 + echo 'Waiting for pulsar namespace to be ready...' Waiting for pulsar namespace to be ready... + i=0 + /usr/local/pulsar/bin/pulsar-admin namespaces list public ++ date + echo '[Fri Apr 4 15:34:09 CST 2025] <<<<<< pulsar is ready >>>>>>' [Fri Apr 4 15:34:09 CST 2025] <<<<<< pulsar is ready >>>>>> [Fri Apr 4 15:34:09 CST 2025] <<<<<< START cdc server in changefeed_finish case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_finish.32723274.out server --log-file /tmp/tidb_cdc_test/changefeed_finish/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_finish/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff successfully + set +x public/default ++ date + echo '[Fri Apr 4 15:34:09 CST 2025] <<<<<< pulsar is ready >>>>>>' [Fri Apr 4 15:34:09 CST 2025] <<<<<< pulsar is ready >>>>>> + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.owner_resign.cli.3364.out cli changefeed create '--sink-uri=pulsar://127.0.0.1:6650/ticdc-owner-resign-test-22328?protocol=canal-json&enable-tidb-extension=true' --server=127.0.0.1:8301 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.capture_session_done_during_task.cli.3386.out cli tso query --pd=http://127.0.0.1:2379 + nc -z localhost 6650 + i=2 + '[' 2 -gt 20 ']' + sleep 2 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489358005347271568 TiDB Cluster ID. tikv_gc_leader_uuid 657fee2d61c000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-4ttz-989qr, pid:4796, start at 2025-04-04 15:34:10.079637695 +0800 CST m=+1.717167573 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:36:10.088 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:34:10.055 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:24:10.055 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489358005347271568 TiDB Cluster ID. tikv_gc_leader_uuid 657fee2d61c000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-4ttz-989qr, pid:4796, start at 2025-04-04 15:34:10.079637695 +0800 CST m=+1.717167573 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:36:10.088 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:34:10.055 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:24:10.055 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489358005715752002 TiDB Cluster ID. tikv_gc_leader_uuid 657fee2d5400009 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-4ttz-989qr, pid:4866, start at 2025-04-04 15:34:10.013805762 +0800 CST m=+1.589371103 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:36:10.021 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:34:10.000 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:24:10.000 +0800 All versions after safe point can be accessed. (DO NOT EDIT) + nc -z localhost 6651 + echo 'Waiting for pulsar namespace to be ready...' Waiting for pulsar namespace to be ready... + i=0 + /usr/local/pulsar/bin/pulsar-admin namespaces list public Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/changefeed_error/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/changefeed_error/tiflash/log/error.log arg matches is ArgMatches { args: {"pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/changefeed_error/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/changefeed_error/tiflash-proxy.toml"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/changefeed_error/tiflash/log/proxy.log"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + nc -z localhost 6650 + i=2 + '[' 2 -gt 20 ']' + sleep 2 public/default Create changefeed successfully! ID: d22c793b-29dd-44f2-a654-64be33d3d261 Info: {"upstream_id":7489357950171925030,"namespace":"default","id":"d22c793b-29dd-44f2-a654-64be33d3d261","sink_uri":"pulsar://127.0.0.1:6650/ticdc-owner-resign-test-22328?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2025-04-04T15:34:10.828267923+08:00","start_ts":457114137444483077,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"pulsar_config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-10-ga306f4894","resolved_ts":457114137444483077,"checkpoint_ts":457114137444483077,"checkpoint_time":"2025-04-04 15:34:10.188"} PASS + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_capture.cli.9862.out cli tso query --pd=http://127.0.0.1:2379 coverage: 2.5% of statements in github.com/pingcap/tiflow/... + nc -z localhost 6650 + echo 'Waiting for pulsar namespace to be ready...' Waiting for pulsar namespace to be ready... + i=0 + /usr/local/pulsar/bin/pulsar-admin namespaces list public check diff failed 2-th time, retry later ++ date + echo '[Fri Apr 4 15:34:11 CST 2025] <<<<<< pulsar is ready >>>>>>' [Fri Apr 4 15:34:11 CST 2025] <<<<<< pulsar is ready >>>>>> + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.processor_resolved_ts_fallback.cli.3343.out cli changefeed create '--sink-uri=pulsar://127.0.0.1:6650/ticdc-processor-resolved-ts-fallback-test-16927?protocol=canal-json&enable-tidb-extension=true' --server=127.0.0.1:8301 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/partition_table/run.sh using Sink-Type: pulsar... <<================= The 1 times to try to start tidb cluster... < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 07:34:11 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [4015 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/ddl-manager {UpstreamID:7489357833116153411 Namespace:default ID:ddl-manager SinkURI:pulsar://127.0.0.1:6650/ticdc-ddl-mamager-test-6367?protocol=canal-json&enable-tidb-extension=true CreateTime:2025-04-04 15:33:42.333112167 +0800 CST StartTs:457114129989369859 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0018d4f30 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-10-ga306f4894 Epoch:457114130133549060} {CheckpointTs:457114130919981071 MinTableBarrierTs:457114130919981071 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7caeff6d-5142-4d33-b057-597856bd1ae2 {"id":"7caeff6d-5142-4d33-b057-597856bd1ae2","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752044} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffb7ffa74d 7caeff6d-5142-4d33-b057-597856bd1ae2 /tidb/cdc/default/default/changefeed/info/ddl-manager {"upstream-id":7489357833116153411,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"pulsar://127.0.0.1:6650/ticdc-ddl-mamager-test-6367?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2025-04-04T15:33:42.333112167+08:00","start-ts":457114129989369859,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"pulsar-config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-10-ga306f4894","epoch":457114130133549060} /tidb/cdc/default/default/changefeed/status/ddl-manager {"checkpoint-ts":457114131051053074,"min-table-barrier-ts":457114131051053074,"admin-job-type":0} /tidb/cdc/default/default/task/position/7caeff6d-5142-4d33-b057-597856bd1ae2/ddl-manager {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489357833116153411 {"id":7489357833116153411,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-manager {UpstreamID:7489357833116153411 Namespace:default ID:ddl-manager SinkURI:pulsar://127.0.0.1:6650/ticdc-ddl-mamager-test-6367?protocol=canal-json&enable-tidb-extension=true CreateTime:2025-04-04 15:33:42.333112167 +0800 CST StartTs:457114129989369859 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0018d4f30 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-10-ga306f4894 Epoch:457114130133549060} {CheckpointTs:457114130919981071 MinTableBarrierTs:457114130919981071 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7caeff6d-5142-4d33-b057-597856bd1ae2 {"id":"7caeff6d-5142-4d33-b057-597856bd1ae2","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752044} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffb7ffa74d 7caeff6d-5142-4d33-b057-597856bd1ae2 /tidb/cdc/default/default/changefeed/info/ddl-manager {"upstream-id":7489357833116153411,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"pulsar://127.0.0.1:6650/ticdc-ddl-mamager-test-6367?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2025-04-04T15:33:42.333112167+08:00","start-ts":457114129989369859,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"pulsar-config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-10-ga306f4894","epoch":457114130133549060} /tidb/cdc/default/default/changefeed/status/ddl-manager {"checkpoint-ts":457114131051053074,"min-table-barrier-ts":457114131051053074,"admin-job-type":0} /tidb/cdc/default/default/task/position/7caeff6d-5142-4d33-b057-597856bd1ae2/ddl-manager {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489357833116153411 {"id":7489357833116153411,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-manager {UpstreamID:7489357833116153411 Namespace:default ID:ddl-manager SinkURI:pulsar://127.0.0.1:6650/ticdc-ddl-mamager-test-6367?protocol=canal-json&enable-tidb-extension=true CreateTime:2025-04-04 15:33:42.333112167 +0800 CST StartTs:457114129989369859 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0018d4f30 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-10-ga306f4894 Epoch:457114130133549060} {CheckpointTs:457114130919981071 MinTableBarrierTs:457114130919981071 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7caeff6d-5142-4d33-b057-597856bd1ae2 {"id":"7caeff6d-5142-4d33-b057-597856bd1ae2","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752044} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffb7ffa74d 7caeff6d-5142-4d33-b057-597856bd1ae2 /tidb/cdc/default/default/changefeed/info/ddl-manager {"upstream-id":7489357833116153411,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"pulsar://127.0.0.1:6650/ticdc-ddl-mamager-test-6367?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2025-04-04T15:33:42.333112167+08:00","start-ts":457114129989369859,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"pulsar-config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-10-ga306f4894","epoch":457114130133549060} /tidb/cdc/default/default/changefeed/status/ddl-manager {"checkpoint-ts":457114131051053074,"min-table-barrier-ts":457114131051053074,"admin-job-type":0} /tidb/cdc/default/default/task/position/7caeff6d-5142-4d33-b057-597856bd1ae2/ddl-manager {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489357833116153411 {"id":7489357833116153411,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x table ddl_manager.finish_mark not exists for 1-th check, retry later + set +x + tso='457114137473581057 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 457114137473581057 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 4 15:34:11 CST 2025] <<<<<< START cdc server in capture_session_done_during_task case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/processor/processorManagerHandleNewChangefeedDelay=sleep(2000)' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.capture_session_done_during_task.34323434.out server --log-file /tmp/tidb_cdc_test/capture_session_done_during_task/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/capture_session_done_during_task/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Create changefeed successfully! ID: 904b7e06-55cd-4f9e-88a5-02c76402c3e0 Info: {"upstream_id":7489357962550911928,"namespace":"default","id":"904b7e06-55cd-4f9e-88a5-02c76402c3e0","sink_uri":"pulsar://127.0.0.1:6650/ticdc-processor-resolved-ts-fallback-test-16927?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2025-04-04T15:34:12.082148161+08:00","start_ts":457114137776881666,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"pulsar_config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-10-ga306f4894","resolved_ts":457114137776881666,"checkpoint_ts":457114137776881666,"checkpoint_time":"2025-04-04 15:34:11.456"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... + set +x [Fri Apr 4 15:34:12 CST 2025] <<<<<< START Pulsar consumer in owner_resign case >>>>>> public/default + nc -z localhost 6650 + echo 'Waiting for pulsar namespace to be ready...' Waiting for pulsar namespace to be ready... + i=0 + /usr/local/pulsar/bin/pulsar-admin namespaces list public + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 07:34:12 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [817 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/dbf3fc4e-80d9-4660-9081-aa2653daa2e6 {"id":"dbf3fc4e-80d9-4660-9081-aa2653daa2e6","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752049} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffb87043f5 dbf3fc4e-80d9-4660-9081-aa2653daa2e6 /tidb/cdc/default/default/upstream/7489357966190138613 {"id":7489357966190138613,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/dbf3fc4e-80d9-4660-9081-aa2653daa2e6 {"id":"dbf3fc4e-80d9-4660-9081-aa2653daa2e6","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752049} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffb87043f5 dbf3fc4e-80d9-4660-9081-aa2653daa2e6 /tidb/cdc/default/default/upstream/7489357966190138613 {"id":7489357966190138613,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + grep -q 'etcd info' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/dbf3fc4e-80d9-4660-9081-aa2653daa2e6 {"id":"dbf3fc4e-80d9-4660-9081-aa2653daa2e6","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752049} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffb87043f5 dbf3fc4e-80d9-4660-9081-aa2653daa2e6 /tidb/cdc/default/default/upstream/7489357966190138613 {"id":7489357966190138613,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x + set +x + tso='457114137710034945 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 457114137710034945 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x ++ date + echo '[Fri Apr 4 15:34:12 CST 2025] <<<<<< pulsar is ready >>>>>>' [Fri Apr 4 15:34:12 CST 2025] <<<<<< pulsar is ready >>>>>> + nc -z localhost 6650 + echo 'Waiting for pulsar namespace to be ready...' Waiting for pulsar namespace to be ready... + i=0 + /usr/local/pulsar/bin/pulsar-admin namespaces list public + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.6172.out cli tso query --pd=http://127.0.0.1:2379 [Fri Apr 4 15:34:13 CST 2025] <<<<<< START Pulsar consumer in changefeed_finish case >>>>>> cdc cli processor list --server http://127.0.0.1:8301 |jq '.|length'|grep -E '^1$' 1 run task successfully table owner_resign.t1 exists check diff failed 1-th time, retry later + set +x [Fri Apr 4 15:34:13 CST 2025] <<<<<< START Pulsar consumer in processor_resolved_ts_fallback case >>>>>> cdc cli processor list --server http://127.0.0.1:8301 |jq '.|length'|grep -E '^1$' table ddl_manager.finish_mark not exists for 2-th check, retry later ***************** properties ***************** "readproportion"="0" "updateproportion"="0" "threadcount"="2" "operationcount"="0" "mysql.host"="127.0.0.1" "mysql.user"="root" "mysql.db"="multi_capture_1" "requestdistribution"="uniform" "recordcount"="10" "scanproportion"="0" "workload"="core" "dotransactions"="false" "insertproportion"="0" "mysql.port"="4000" "readallfields"="true" ********************************************** Run finished, takes 9.296958ms INSERT - Takes(s): 0.0, Count: 10, OPS: 2107.6, Avg(us): 1799, Min(us): 970, Max(us): 4475, 95th(us): 5000, 99th(us): 5000 Tenant not found Reason: Tenant not found Create changefeed successfully! ID: 118b8a9e-88b3-4aa4-bfef-20d7eb4a8fb4 Info: {"upstream_id":7489357955358834369,"namespace":"default","id":"118b8a9e-88b3-4aa4-bfef-20d7eb4a8fb4","sink_uri":"pulsar+ssl://127.0.0.1:6651/ticdc-tiflash-test-9313?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2025-04-04T15:34:13.658588895+08:00","start_ts":457114135156228097,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"pulsar_config":{"tls-trust-certs-file-path":"/tmp/tidb_cdc_test/tiflash/ca.cert.pem","connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30,"auth-tls-certificate-path":"/tmp/tidb_cdc_test/tiflash/broker_client.cert.pem","auth-tls-private-key-path":"/tmp/tidb_cdc_test/tiflash/broker_client.key-pk8.pem"},"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-10-ga306f4894","resolved_ts":457114135156228097,"checkpoint_ts":457114135156228097,"checkpoint_time":"2025-04-04 15:34:01.459"} [Fri Apr 4 15:34:13 CST 2025] <<<<<< START Pulsar consumer in tiflash case >>>>>> check diff failed 3-th time, retry later check diff failed 1-th time, retry later 1 run task successfully table processor_resolved_ts_fallback.t1 not exists for 1-th check, retry later table cdc_tiflash_test.multi_data_type not exists for 1-th check, retry later public/default + i=1 + '[' 1 -gt 20 ']' + sleep 2 public/default + set +x + tso='457114138209157121 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 457114138209157121 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x ++ date + echo '[Fri Apr 4 15:34:14 CST 2025] <<<<<< pulsar is ready >>>>>>' [Fri Apr 4 15:34:14 CST 2025] <<<<<< pulsar is ready >>>>>> + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.batch_add_table.cli.3427.out cli changefeed create '--sink-uri=pulsar://127.0.0.1:6650/ticdc-batch-add-table-test-12122?protocol=canal-json&enable-tidb-extension=true' start tidb cluster in /tmp/tidb_cdc_test/partition_table Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... ***************** properties ***************** "mysql.db"="multi_capture_2" "workload"="core" "insertproportion"="0" "mysql.port"="4000" "updateproportion"="0" "dotransactions"="false" "recordcount"="10" "operationcount"="0" "mysql.host"="127.0.0.1" "scanproportion"="0" "requestdistribution"="uniform" "mysql.user"="root" "readallfields"="true" "readproportion"="0" "threadcount"="2" ********************************************** Run finished, takes 9.508572ms INSERT - Takes(s): 0.0, Count: 10, OPS: 2049.5, Avg(us): 1837, Min(us): 1035, Max(us): 4571, 95th(us): 5000, 99th(us): 5000 ++ date + echo '[Fri Apr 4 15:34:14 CST 2025] <<<<<< pulsar is ready >>>>>>' [Fri Apr 4 15:34:14 CST 2025] <<<<<< pulsar is ready >>>>>> + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.resolve_lock.cli.3432.out cli changefeed create --start-ts=457114135811325953 '--sink-uri=pulsar://127.0.0.1:6650/ticdc-resolve-lock-test-11953?protocol=canal-json&enable-tidb-extension=true' + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 07:34:14 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [817 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/72509ea9-59b7-49bc-ba4b-05b4deab8cd7 {"id":"72509ea9-59b7-49bc-ba4b-05b4deab8cd7","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752052} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffb8734614 72509ea9-59b7-49bc-ba4b-05b4deab8cd7 /tidb/cdc/default/default/upstream/7489357962719330633 {"id":7489357962719330633,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/72509ea9-59b7-49bc-ba4b-05b4deab8cd7 {"id":"72509ea9-59b7-49bc-ba4b-05b4deab8cd7","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752052} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffb8734614 72509ea9-59b7-49bc-ba4b-05b4deab8cd7 /tidb/cdc/default/default/upstream/7489357962719330633 {"id":7489357962719330633,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/72509ea9-59b7-49bc-ba4b-05b4deab8cd7 {"id":"72509ea9-59b7-49bc-ba4b-05b4deab8cd7","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752052} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffb8734614 72509ea9-59b7-49bc-ba4b-05b4deab8cd7 /tidb/cdc/default/default/upstream/7489357962719330633 {"id":7489357962719330633,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Fri Apr 4 15:34:14 CST 2025] <<<<<< START Pulsar consumer in canal_json_content_compatible case >>>>>> table ddl_manager.finish_mark not exists for 3-th check, retry later check diff failed 4-th time, retry later Create changefeed successfully! ID: bdead28d-3cdd-4b72-8867-e8bc95c0e49a Info: {"upstream_id":7489357967512829777,"namespace":"default","id":"bdead28d-3cdd-4b72-8867-e8bc95c0e49a","sink_uri":"pulsar://127.0.0.1:6650/ticdc-batch-add-table-test-12122?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2025-04-04T15:34:15.490839723+08:00","start_ts":457114138687307780,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"pulsar_config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-10-ga306f4894","resolved_ts":457114138687307780,"checkpoint_ts":457114138687307780,"checkpoint_time":"2025-04-04 15:34:14.929"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... table processor_resolved_ts_fallback.t1 exists [Fri Apr 4 15:34:15 CST 2025] <<<<<< START cdc server in processor_resolved_ts_fallback case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.processor_resolved_ts_fallback.35053507.out server --log-file /tmp/tidb_cdc_test/processor_resolved_ts_fallback/cdc2.log --log-level debug --data-dir /tmp/tidb_cdc_test/processor_resolved_ts_fallback/cdc_data2 --cluster-id default --addr 127.0.0.1:8302 --pd http://127.0.0.1:2379 + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8302 failed: Connection refused * Failed to connect to 127.0.0.1 port 8302: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table cdc_tiflash_test.multi_data_type exists ***************** properties ***************** "scanproportion"="0" "operationcount"="0" "mysql.user"="root" "threadcount"="4" "insertproportion"="0" "requestdistribution"="uniform" "updateproportion"="0" "readallfields"="true" "mysql.db"="changefeed_error" "recordcount"="20" "mysql.host"="127.0.0.1" "readproportion"="0" "dotransactions"="false" "workload"="core" "mysql.port"="4000" ********************************************** Run finished, takes 10.998916ms INSERT - Takes(s): 0.0, Count: 20, OPS: 3403.9, Avg(us): 2081, Min(us): 1109, Max(us): 5162, 95th(us): 6000, 99th(us): 6000 [Fri Apr 4 15:34:15 CST 2025] <<<<<< START cdc server in changefeed_error case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/NewChangefeedNoRetryError=1*return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.62226226.out server --log-file /tmp/tidb_cdc_test/changefeed_error/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_error/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ***************** properties ***************** "updateproportion"="0" "mysql.db"="multi_capture_3" "recordcount"="10" "mysql.port"="4000" "readallfields"="true" "dotransactions"="false" "mysql.host"="127.0.0.1" "insertproportion"="0" "scanproportion"="0" "requestdistribution"="uniform" "workload"="core" "readproportion"="0" "mysql.user"="root" "threadcount"="2" "operationcount"="0" ********************************************** Run finished, takes 9.258224ms INSERT - Takes(s): 0.0, Count: 10, OPS: 1990.7, Avg(us): 1750, Min(us): 968, Max(us): 4116, 95th(us): 5000, 99th(us): 5000 Create changefeed successfully! ID: f3ff740d-a03e-4ba8-9f30-9a27d416acf8 Info: {"upstream_id":7489357969709569958,"namespace":"default","id":"f3ff740d-a03e-4ba8-9f30-9a27d416acf8","sink_uri":"pulsar://127.0.0.1:6650/ticdc-resolve-lock-test-11953?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2025-04-04T15:34:15.791573777+08:00","start_ts":457114135811325953,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"pulsar_config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-10-ga306f4894","resolved_ts":457114135811325953,"checkpoint_ts":457114135811325953,"checkpoint_time":"2025-04-04 15:34:03.958"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... table test.finish_mark not exists for 1-th check, retry later check diff failed 2-th time, retry later check diff failed 1-th time, retry later + /usr/local/pulsar/bin/pulsar-admin namespaces list public check diff failed 2-th time, retry later ***************** properties ***************** "requestdistribution"="uniform" "scanproportion"="0" "recordcount"="10" "mysql.host"="127.0.0.1" "dotransactions"="false" "mysql.db"="multi_capture_4" "updateproportion"="0" "workload"="core" "threadcount"="2" "readallfields"="true" "readproportion"="0" "operationcount"="0" "insertproportion"="0" "mysql.port"="4000" "mysql.user"="root" ********************************************** Run finished, takes 8.955827ms INSERT - Takes(s): 0.0, Count: 10, OPS: 2147.1, Avg(us): 1686, Min(us): 972, Max(us): 4180, 95th(us): 5000, 99th(us): 5000 [Fri Apr 4 15:34:16 CST 2025] <<<<<< START cdc server in multi_capture case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_capture.99909992.out server --log-file /tmp/tidb_cdc_test/multi_capture/cdc1.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_capture/cdc_data1 --cluster-id default --addr 127.0.0.1:8301 + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8301 failed: Connection refused * Failed to connect to 127.0.0.1 port 8301: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [Fri Apr 4 15:34:16 CST 2025] <<<<<< START Pulsar consumer in capture_session_done_during_task case >>>>>> lease 223195ffb8734614 revoked + set +x [Fri Apr 4 15:34:16 CST 2025] <<<<<< START Pulsar consumer in batch_add_table case >>>>>> table batch_add_table.finish_mark not exists for 1-th check, retry later + set +x [Fri Apr 4 15:34:17 CST 2025] <<<<<< START Pulsar consumer in resolve_lock case >>>>>> go: downloading go1.23.4 (linux/amd64) table ddl_manager.finish_mark not exists for 4-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table capture_session_done_during_task.t exists check diff failed 5-th time, retry later check diff failed 1-th time, retry later check diff successfully [Fri Apr 4 15:34:17 CST 2025] <<<<<< START cdc server in owner_resign case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ChangefeedOwnerDontUpdateCheckpoint=return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.owner_resign.36213623.out server --log-file /tmp/tidb_cdc_test/owner_resign/cdc2.log --log-level debug --data-dir /tmp/tidb_cdc_test/owner_resign/cdc_data2 --cluster-id default --addr 127.0.0.1:8302 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8302 failed: Connection refused * Failed to connect to 127.0.0.1 port 8302: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 public/default check diff successfully ++ date + echo '[Fri Apr 4 15:34:18 CST 2025] <<<<<< pulsar is ready >>>>>>' [Fri Apr 4 15:34:18 CST 2025] <<<<<< pulsar is ready >>>>>> + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.simple.cli.3500.out cli changefeed create --start-ts=457114135301718017 '--sink-uri=pulsar://127.0.0.1:6650/ticdc-simple-test-19740?protocol=canal-json&enable-tidb-extension=true' table test.finish_mark not exists for 2-th check, retry later check diff successfully wait process cdc.test exit for 1-th time... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8302 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8302 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 07:34:18 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3971 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/904b7e06-55cd-4f9e-88a5-02c76402c3e0 {UpstreamID:7489357962550911928 Namespace:default ID:904b7e06-55cd-4f9e-88a5-02c76402c3e0 SinkURI:pulsar://127.0.0.1:6650/ticdc-processor-resolved-ts-fallback-test-16927?protocol=canal-json&enable-tidb-extension=true CreateTime:2025-04-04 15:34:12.082148161 +0800 CST StartTs:457114137776881666 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003cba1b0 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-10-ga306f4894 Epoch:457114137934168067} {CheckpointTs:457114139585937416 MinTableBarrierTs:457114139585937416 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/81628c0e-4437-4a59-930e-7cef65ba605f {"id":"81628c0e-4437-4a59-930e-7cef65ba605f","address":"127.0.0.1:8302","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752056} /tidb/cdc/default/__cdc_meta__/capture/e961eba9-a1b2-4f89-8df7-d4d4e3c56689 {"id":"e961eba9-a1b2-4f89-8df7-d4d4e3c56689","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752042} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffb871bccf e961eba9-a1b2-4f89-8df7-d4d4e3c56689 /tidb/cdc/default/__cdc_meta__/owner/223195ffb871bd76 81628c0e-4437-4a59-930e-7cef65ba605f /tidb/cdc/default/default/changefeed/info/904b7e06-55cd-4f9e-88a5-02c76402c3e0 {"upstream-id":7489357962550911928,"namespace":"default","changefeed-id":"904b7e06-55cd-4f9e-88a5-02c76402c3e0","sink-uri":"pulsar://127.0.0.1:6650/ticdc-processor-resolved-ts-fallback-test-16927?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2025-04-04T15:34:12.082148161+08:00","start-ts":457114137776881666,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"pulsar-config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-10-ga306f4894","epoch":457114137934168067} /tidb/cdc/default/default/changefeed/status/904b7e06-55cd-4f9e-88a5-02c76402c3e0 {"checkpoint-ts":457114139585937416,"min-table-barrier-ts":457114139585937416,"admin-job-type":0} /tidb/cdc/default/default/task/position/81628c0e-4437-4a59-930e-7cef65ba605f/904b7e06-55cd-4f9e-88a5-02c76402c3e0 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/e961eba9-a1b2-4f89-8df7-d4d4e3c56689/904b7e06-55cd-4f9e-88a5-02c76402c3e0 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489357962550911928 {"id":7489357962550911928,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/904b7e06-55cd-4f9e-88a5-02c76402c3e0 {UpstreamID:7489357962550911928 Namespace:default ID:904b7e06-55cd-4f9e-88a5-02c76402c3e0 SinkURI:pulsar://127.0.0.1:6650/ticdc-processor-resolved-ts-fallback-test-16927?protocol=canal-json&enable-tidb-extension=true CreateTime:2025-04-04 15:34:12.082148161 +0800 CST StartTs:457114137776881666 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003cba1b0 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-10-ga306f4894 Epoch:457114137934168067} {CheckpointTs:457114139585937416 MinTableBarrierTs:457114139585937416 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/81628c0e-4437-4a59-930e-7cef65ba605f {"id":"81628c0e-4437-4a59-930e-7cef65ba605f","address":"127.0.0.1:8302","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752056} /tidb/cdc/default/__cdc_meta__/capture/e961eba9-a1b2-4f89-8df7-d4d4e3c56689 {"id":"e961eba9-a1b2-4f89-8df7-d4d4e3c56689","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752042} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffb871bccf e961eba9-a1b2-4f89-8df7-d4d4e3c56689 /tidb/cdc/default/__cdc_meta__/owner/223195ffb871bd76 81628c0e-4437-4a59-930e-7cef65ba605f /tidb/cdc/default/default/changefeed/info/904b7e06-55cd-4f9e-88a5-02c76402c3e0 {"upstream-id":7489357962550911928,"namespace":"default","changefeed-id":"904b7e06-55cd-4f9e-88a5-02c76402c3e0","sink-uri":"pulsar://127.0.0.1:6650/ticdc-processor-resolved-ts-fallback-test-16927?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2025-04-04T15:34:12.082148161+08:00","start-ts":457114137776881666,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"pulsar-config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-10-ga306f4894","epoch":457114137934168067} /tidb/cdc/default/default/changefeed/status/904b7e06-55cd-4f9e-88a5-02c76402c3e0 {"checkpoint-ts":457114139585937416,"min-table-barrier-ts":457114139585937416,"admin-job-type":0} /tidb/cdc/default/default/task/position/81628c0e-4437-4a59-930e-7cef65ba605f/904b7e06-55cd-4f9e-88a5-02c76402c3e0 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/e961eba9-a1b2-4f89-8df7-d4d4e3c56689/904b7e06-55cd-4f9e-88a5-02c76402c3e0 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489357962550911928 {"id":7489357962550911928,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: changefeedID: default/904b7e06-55cd-4f9e-88a5-02c76402c3e0 {UpstreamID:7489357962550911928 Namespace:default ID:904b7e06-55cd-4f9e-88a5-02c76402c3e0 SinkURI:pulsar://127.0.0.1:6650/ticdc-processor-resolved-ts-fallback-test-16927?protocol=canal-json&enable-tidb-extension=true CreateTime:2025-04-04 15:34:12.082148161 +0800 CST StartTs:457114137776881666 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003cba1b0 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-10-ga306f4894 Epoch:457114137934168067} {CheckpointTs:457114139585937416 MinTableBarrierTs:457114139585937416 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/81628c0e-4437-4a59-930e-7cef65ba605f {"id":"81628c0e-4437-4a59-930e-7cef65ba605f","address":"127.0.0.1:8302","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752056} /tidb/cdc/default/__cdc_meta__/capture/e961eba9-a1b2-4f89-8df7-d4d4e3c56689 {"id":"e961eba9-a1b2-4f89-8df7-d4d4e3c56689","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752042} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffb871bccf e961eba9-a1b2-4f89-8df7-d4d4e3c56689 /tidb/cdc/default/__cdc_meta__/owner/223195ffb871bd76 81628c0e-4437-4a59-930e-7cef65ba605f /tidb/cdc/default/default/changefeed/info/904b7e06-55cd-4f9e-88a5-02c76402c3e0 {"upstream-id":7489357962550911928,"namespace":"default","changefeed-id":"904b7e06-55cd-4f9e-88a5-02c76402c3e0","sink-uri":"pulsar://127.0.0.1:6650/ticdc-processor-resolved-ts-fallback-test-16927?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2025-04-04T15:34:12.082148161+08:00","start-ts":457114137776881666,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"pulsar-config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-10-ga306f4894","epoch":457114137934168067} /tidb/cdc/default/default/changefeed/status/904b7e06-55cd-4f9e-88a5-02c76402c3e0 {"checkpoint-ts":457114139585937416,"min-table-barrier-ts":457114139585937416,"admin-job-type":0} /tidb/cdc/default/default/task/position/81628c0e-4437-4a59-930e-7cef65ba605f/904b7e06-55cd-4f9e-88a5-02c76402c3e0 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/e961eba9-a1b2-4f89-8df7-d4d4e3c56689/904b7e06-55cd-4f9e-88a5-02c76402c3e0 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489357962550911928 {"id":7489357962550911928,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x table processor_resolved_ts_fallback.t2 not exists for 1-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 07:34:18 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [817 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/51c50a27-0a75-49c0-be9b-159863c14d2e {"id":"51c50a27-0a75-49c0-be9b-159863c14d2e","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752055} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffb896d6ea 51c50a27-0a75-49c0-be9b-159863c14d2e /tidb/cdc/default/default/upstream/7489358005347271568 {"id":7489358005347271568,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/51c50a27-0a75-49c0-be9b-159863c14d2e {"id":"51c50a27-0a75-49c0-be9b-159863c14d2e","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752055} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffb896d6ea 51c50a27-0a75-49c0-be9b-159863c14d2e /tidb/cdc/default/default/upstream/7489358005347271568 {"id":7489358005347271568,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/51c50a27-0a75-49c0-be9b-159863c14d2e {"id":"51c50a27-0a75-49c0-be9b-159863c14d2e","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752055} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffb896d6ea 51c50a27-0a75-49c0-be9b-159863c14d2e /tidb/cdc/default/default/upstream/7489358005347271568 {"id":7489358005347271568,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x ++ date + echo '[Fri Apr 4 15:34:18 CST 2025] <<<<<< START pulsar cluster in changefeed_error case >>>>>>' [Fri Apr 4 15:34:18 CST 2025] <<<<<< START pulsar cluster in changefeed_error case >>>>>> + workdir=/tmp/tidb_cdc_test/changefeed_error + cluster_type=normal + cd /tmp/tidb_cdc_test/changefeed_error + DEFAULT_PULSAR_HOME=/usr/local/pulsar + pulsar_dir=/usr/local/pulsar ++ cat + mtls_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderTls brokerClientTlsEnabled=true brokerClientTrustCertsFilePath=/tmp/tidb_cdc_test/changefeed_error/ca.cert.pem brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls brokerClientAuthenticationParameters={"tlsCertFile":"/tmp/tidb_cdc_test/changefeed_error/broker_client.cert.pem","tlsKeyFile":"/tmp/tidb_cdc_test/changefeed_error/broker_client.key-pk8.pem"} brokerServicePortTls=6651 webServicePortTls=8443 tlsTrustCertsFilePath=/tmp/tidb_cdc_test/changefeed_error/ca.cert.pem tlsCertificateFilePath=/tmp/tidb_cdc_test/changefeed_error/server.cert.pem tlsKeyFilePath=/tmp/tidb_cdc_test/changefeed_error/server.key-pk8.pem tlsRequireTrustedClientCertOnConnect=true tlsAllowInsecureConnection=false tlsCertRefreshCheckDurationSec=300' ++ cat + normal_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' ++ cat + mtls_client_conf=' webServiceUrl=https://localhost:8443/ brokerServiceUrl=pulsar+ssl://localhost:6651/ authPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls authParams=tlsCertFile:/tmp/tidb_cdc_test/changefeed_error/broker_client.cert.pem,tlsKeyFile:/tmp/tidb_cdc_test/changefeed_error/broker_client.key-pk8.pem tlsTrustCertsFilePath=/tmp/tidb_cdc_test/changefeed_error/ca.cert.pem' ++ cat + oauth_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/ authPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 authParams={"privateKey":"/tmp/tidb_cdc_test/changefeed_error/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"}' ++ cat + oauth_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderToken brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 brokerClientAuthenticationParameters={"privateKey":"file:///tmp/tidb_cdc_test/changefeed_error/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"} tokenSecretKey=data:;base64,U0poWDM2X0thcFlTeWJCdEpxMzVseFhfQnJyNExSVVJTa203UW1YSkdteThwVUZXOUVJT2NWUVBzeWt6OS1qag==' ++ cat + credential_json=' { "client_id":"1234", "client_secret":"e0KVlA2EiBfjoN13olyZd2kv1KL", "audience":"cdc-api-uri", "issuer_url":"http://localhost:9096", "type": "client_credentials" }' ++ cat + cert_server_conf='[ req ] default_bits = 2048 prompt = no default_md = sha256 distinguished_name = dn [ v3_ext ] authorityKeyIdentifier=keyid,issuer:always basicConstraints=CA:FALSE keyUsage=critical, digitalSignature, keyEncipherment extendedKeyUsage=serverAuth subjectAltName=@alt_names [ dn ] CN = server [ alt_names ] DNS.1 = localhost IP.1 = 127.0.0.1' + echo ' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' + cp /usr/local/pulsar/conf/standalone.conf /tmp/tidb_cdc_test/changefeed_error/pulsar_standalone.conf + pulsar_port=6650 + '[' normal == mtls ']' + '[' normal == oauth ']' + echo 'no cluster type specified, using default configuration.' no cluster type specified, using default configuration. ++ date + echo '[Fri Apr 4 15:34:18 CST 2025] <<<<<< START pulsar cluster in normal mode in changefeed_error case >>>>>>' [Fri Apr 4 15:34:18 CST 2025] <<<<<< START pulsar cluster in normal mode in changefeed_error case >>>>>> + echo 'Waiting for pulsar port to be ready...' Waiting for pulsar port to be ready... + i=0 + /usr/local/pulsar/bin/pulsar standalone --config /tmp/tidb_cdc_test/changefeed_error/pulsar_standalone.conf -nfw --metadata-dir /tmp/tidb_cdc_test/changefeed_error/pulsar-metadata --bookkeeper-dir /tmp/tidb_cdc_test/changefeed_error/pulsar-bookie + nc -z localhost 6650 + i=1 + '[' 1 -gt 20 ']' + sleep 2 wait process cdc.test exit for 2-th time... Create changefeed successfully! ID: 14695d41-99a4-4e3e-9f22-16e029640d18 Info: {"upstream_id":7489357950984869366,"namespace":"default","id":"14695d41-99a4-4e3e-9f22-16e029640d18","sink_uri":"pulsar://127.0.0.1:6650/ticdc-simple-test-19740?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2025-04-04T15:34:19.147229671+08:00","start_ts":457114135301718017,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"pulsar_config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-10-ga306f4894","resolved_ts":457114135301718017,"checkpoint_ts":457114135301718017,"checkpoint_time":"2025-04-04 15:34:02.014"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 4 15:34:19 CST 2025] <<<<<< run test case tiflash success! >>>>>> table batch_add_table.finish_mark not exists for 2-th check, retry later table ddl_manager.finish_mark not exists for 5-th check, retry later check diff successfully + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8301 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 07:34:19 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [817 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/64b95cdc-8be7-4ffe-92d2-9beabee4aca6 {"id":"64b95cdc-8be7-4ffe-92d2-9beabee4aca6","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752057} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffb8935038 64b95cdc-8be7-4ffe-92d2-9beabee4aca6 /tidb/cdc/default/default/upstream/7489357992790424245 {"id":7489357992790424245,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/64b95cdc-8be7-4ffe-92d2-9beabee4aca6 {"id":"64b95cdc-8be7-4ffe-92d2-9beabee4aca6","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752057} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffb8935038 64b95cdc-8be7-4ffe-92d2-9beabee4aca6 /tidb/cdc/default/default/upstream/7489357992790424245 {"id":7489357992790424245,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/64b95cdc-8be7-4ffe-92d2-9beabee4aca6 {"id":"64b95cdc-8be7-4ffe-92d2-9beabee4aca6","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752057} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffb8935038 64b95cdc-8be7-4ffe-92d2-9beabee4aca6 /tidb/cdc/default/default/upstream/7489357992790424245 {"id":7489357992790424245,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Fri Apr 4 15:34:19 CST 2025] <<<<<< START cdc server in multi_capture case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_capture.1004710049.out server --log-file /tmp/tidb_cdc_test/multi_capture/cdc2.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_capture/cdc_data2 --cluster-id default --addr 127.0.0.1:8302 + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8302 failed: Connection refused * Failed to connect to 127.0.0.1 port 8302: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff failed 2-th time, retry later table test.finish_mark not exists for 3-th check, retry later + set +x [Fri Apr 4 15:34:20 CST 2025] <<<<<< START Pulsar consumer in simple case >>>>>> succeed to verify meta placement rules check diff failed 1-th time, retry later ERROR 1146 (42S02) at line 1: Table 'test.simple1' doesn't exist check data failed 1-th time, retry later + nc -z localhost 6650 + i=2 + '[' 2 -gt 20 ']' + sleep 2 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8302 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8302 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 07:34:20 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3971 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/d22c793b-29dd-44f2-a654-64be33d3d261 {UpstreamID:7489357950171925030 Namespace:default ID:d22c793b-29dd-44f2-a654-64be33d3d261 SinkURI:pulsar://127.0.0.1:6650/ticdc-owner-resign-test-22328?protocol=canal-json&enable-tidb-extension=true CreateTime:2025-04-04 15:34:10.828267923 +0800 CST StartTs:457114137444483077 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0042925a0 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-10-ga306f4894 Epoch:457114137602031621} {CheckpointTs:457114140000387075 MinTableBarrierTs:457114140000387075 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/47462714-420d-4d04-bd1b-36f6de63a015 {"id":"47462714-420d-4d04-bd1b-36f6de63a015","address":"127.0.0.1:8302","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752058} /tidb/cdc/default/__cdc_meta__/capture/cd58ca7c-c620-4bf4-997f-0744ba25e64f {"id":"cd58ca7c-c620-4bf4-997f-0744ba25e64f","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752040} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffb86df3cc cd58ca7c-c620-4bf4-997f-0744ba25e64f /tidb/cdc/default/__cdc_meta__/owner/223195ffb86df4af 47462714-420d-4d04-bd1b-36f6de63a015 /tidb/cdc/default/default/changefeed/info/d22c793b-29dd-44f2-a654-64be33d3d261 {"upstream-id":7489357950171925030,"namespace":"default","changefeed-id":"d22c793b-29dd-44f2-a654-64be33d3d261","sink-uri":"pulsar://127.0.0.1:6650/ticdc-owner-resign-test-22328?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2025-04-04T15:34:10.828267923+08:00","start-ts":457114137444483077,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"pulsar-config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-10-ga306f4894","epoch":457114137602031621} /tidb/cdc/default/default/changefeed/status/d22c793b-29dd-44f2-a654-64be33d3d261 {"checkpoint-ts":457114140000387075,"min-table-barrier-ts":457114140000387075,"admin-job-type":0} /tidb/cdc/default/default/task/position/47462714-420d-4d04-bd1b-36f6de63a015/d22c793b-29dd-44f2-a654-64be33d3d261 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/cd58ca7c-c620-4bf4-997f-0744ba25e64f/d22c793b-29dd-44f2-a654-64be33d3d261 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489357950171925030 {"id":7489357950171925030,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/d22c793b-29dd-44f2-a654-64be33d3d261 {UpstreamID:7489357950171925030 Namespace:default ID:d22c793b-29dd-44f2-a654-64be33d3d261 SinkURI:pulsar://127.0.0.1:6650/ticdc-owner-resign-test-22328?protocol=canal-json&enable-tidb-extension=true CreateTime:2025-04-04 15:34:10.828267923 +0800 CST StartTs:457114137444483077 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0042925a0 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-10-ga306f4894 Epoch:457114137602031621} {CheckpointTs:457114140000387075 MinTableBarrierTs:457114140000387075 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/47462714-420d-4d04-bd1b-36f6de63a015 {"id":"47462714-420d-4d04-bd1b-36f6de63a015","address":"127.0.0.1:8302","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752058} /tidb/cdc/default/__cdc_meta__/capture/cd58ca7c-c620-4bf4-997f-0744ba25e64f {"id":"cd58ca7c-c620-4bf4-997f-0744ba25e64f","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752040} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffb86df3cc cd58ca7c-c620-4bf4-997f-0744ba25e64f /tidb/cdc/default/__cdc_meta__/owner/223195ffb86df4af 47462714-420d-4d04-bd1b-36f6de63a015 /tidb/cdc/default/default/changefeed/info/d22c793b-29dd-44f2-a654-64be33d3d261 {"upstream-id":7489357950171925030,"namespace":"default","changefeed-id":"d22c793b-29dd-44f2-a654-64be33d3d261","sink-uri":"pulsar://127.0.0.1:6650/ticdc-owner-resign-test-22328?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2025-04-04T15:34:10.828267923+08:00","start-ts":457114137444483077,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"pulsar-config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-10-ga306f4894","epoch":457114137602031621} /tidb/cdc/default/default/changefeed/status/d22c793b-29dd-44f2-a654-64be33d3d261 {"checkpoint-ts":457114140000387075,"min-table-barrier-ts":457114140000387075,"admin-job-type":0} /tidb/cdc/default/default/task/position/47462714-420d-4d04-bd1b-36f6de63a015/d22c793b-29dd-44f2-a654-64be33d3d261 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/cd58ca7c-c620-4bf4-997f-0744ba25e64f/d22c793b-29dd-44f2-a654-64be33d3d261 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489357950171925030 {"id":7489357950171925030,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: changefeedID: default/d22c793b-29dd-44f2-a654-64be33d3d261 {UpstreamID:7489357950171925030 Namespace:default ID:d22c793b-29dd-44f2-a654-64be33d3d261 SinkURI:pulsar://127.0.0.1:6650/ticdc-owner-resign-test-22328?protocol=canal-json&enable-tidb-extension=true CreateTime:2025-04-04 15:34:10.828267923 +0800 CST StartTs:457114137444483077 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0042925a0 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-10-ga306f4894 Epoch:457114137602031621} {CheckpointTs:457114140000387075 MinTableBarrierTs:457114140000387075 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/47462714-420d-4d04-bd1b-36f6de63a015 {"id":"47462714-420d-4d04-bd1b-36f6de63a015","address":"127.0.0.1:8302","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752058} /tidb/cdc/default/__cdc_meta__/capture/cd58ca7c-c620-4bf4-997f-0744ba25e64f {"id":"cd58ca7c-c620-4bf4-997f-0744ba25e64f","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752040} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffb86df3cc cd58ca7c-c620-4bf4-997f-0744ba25e64f /tidb/cdc/default/__cdc_meta__/owner/223195ffb86df4af 47462714-420d-4d04-bd1b-36f6de63a015 /tidb/cdc/default/default/changefeed/info/d22c793b-29dd-44f2-a654-64be33d3d261 {"upstream-id":7489357950171925030,"namespace":"default","changefeed-id":"d22c793b-29dd-44f2-a654-64be33d3d261","sink-uri":"pulsar://127.0.0.1:6650/ticdc-owner-resign-test-22328?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2025-04-04T15:34:10.828267923+08:00","start-ts":457114137444483077,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"pulsar-config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-10-ga306f4894","epoch":457114137602031621} /tidb/cdc/default/default/changefeed/status/d22c793b-29dd-44f2-a654-64be33d3d261 {"checkpoint-ts":457114140000387075,"min-table-barrier-ts":457114140000387075,"admin-job-type":0} /tidb/cdc/default/default/task/position/47462714-420d-4d04-bd1b-36f6de63a015/d22c793b-29dd-44f2-a654-64be33d3d261 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/cd58ca7c-c620-4bf4-997f-0744ba25e64f/d22c793b-29dd-44f2-a654-64be33d3d261 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489357950171925030 {"id":7489357950171925030,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x cdc cli capture list --server http://127.0.0.1:8302 |jq '.|length'|grep -E '^2$' 2 run task successfully % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489358054879327813 TiDB Cluster ID. tikv_gc_leader_uuid 657fee37c200014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-gr29-fxw7k, pid:11415, start at 2025-04-04 15:34:20.728070705 +0800 CST m=+1.581825080 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:36:20.735 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:34:20.729 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:24:20.729 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489358054879327813 TiDB Cluster ID. tikv_gc_leader_uuid 657fee37c200014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-gr29-fxw7k, pid:11415, start at 2025-04-04 15:34:20.728070705 +0800 CST m=+1.581825080 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:36:20.735 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:34:20.729 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:24:20.729 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489358056647547252 TiDB Cluster ID. tikv_gc_leader_uuid 657fee37b3c0007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-gr29-fxw7k, pid:11494, start at 2025-04-04 15:34:20.634652926 +0800 CST m=+1.413372067 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:36:20.643 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:34:20.623 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:24:20.623 +0800 All versions after safe point can be accessed. (DO NOT EDIT) table processor_resolved_ts_fallback.t2 exists table processor_resolved_ts_fallback.t3 not exists for 1-th check, retry later Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/partition_table/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/partition_table/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/partition_table/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/partition_table/tiflash-proxy.toml"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/partition_table/tiflash/log/proxy.log"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table batch_add_table.finish_mark not exists for 3-th check, retry later table ddl_manager.finish_mark not exists for 6-th check, retry later check diff successfully check diff failed 1-th time, retry later table test.finish_mark not exists for 4-th check, retry later go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20241223052309-3735ed55a394 go: downloading github.com/pingcap/kvproto v0.0.0-20241120022153-92b0414aeed8 go: downloading github.com/pingcap/log v1.1.1-0.20241212030209-7e3ff8601a2a go: downloading github.com/pingcap/errors v0.11.5-0.20240318064555-6bd07397691f go: downloading github.com/tikv/client-go/v2 v2.0.8-0.20241209094930-06d7f4b9233b go: downloading github.com/tikv/pd/client v0.0.0-20241111073742-238d4d79ea31 go: downloading github.com/BurntSushi/toml v1.4.1-0.20240526193622-a339e1f7089c go: downloading go.uber.org/zap v1.27.0 go: downloading github.com/pingcap/failpoint v0.0.0-20240528011301-b51a646c7c86 go: downloading gopkg.in/natefinch/lumberjack.v2 v2.2.1 go: downloading go.uber.org/atomic v1.11.0 go: downloading github.com/tikv/pd v1.1.0-beta.0.20240407022249-7179657d129b go: downloading go.uber.org/multierr v1.11.0 go: downloading github.com/pkg/errors v0.9.1 go: downloading github.com/google/uuid v1.6.0 go: downloading github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a go: downloading go.etcd.io/etcd/client/v3 v3.5.12 go: downloading github.com/opentracing/opentracing-go v1.2.0 go: downloading go.etcd.io/etcd/api/v3 v3.5.12 go: downloading google.golang.org/grpc v1.64.0 go: downloading github.com/golang/protobuf v1.5.4 go: downloading github.com/prometheus/client_model v0.6.1 go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading github.com/prometheus/client_golang v1.20.5 go: downloading github.com/twmb/murmur3 v1.1.6 go: downloading github.com/gogo/protobuf v1.3.2 go: downloading golang.org/x/sync v0.10.0 go: downloading github.com/google/btree v1.1.2 go: downloading github.com/dgryski/go-farm v0.0.0-20240924180020-3414d57e47da go: downloading github.com/docker/go-units v0.5.0 go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 go: downloading google.golang.org/protobuf v1.35.2 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec go: downloading google.golang.org/genproto/googleapis/api v0.0.0-20240401170217-c3f982113cda go: downloading github.com/coreos/go-semver v0.3.1 go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.12 go: downloading github.com/coreos/go-systemd/v22 v22.5.0 go: downloading google.golang.org/genproto v0.0.0-20240401170217-c3f982113cda go: downloading github.com/beorn7/perks v1.0.1 go: downloading github.com/prometheus/procfs v0.15.1 go: downloading github.com/prometheus/common v0.61.0 go: downloading github.com/cespare/xxhash/v2 v2.3.0 go: downloading github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 go: downloading golang.org/x/sys v0.28.0 go: downloading golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 go: downloading github.com/cloudfoundry/gosigar v1.3.6 go: downloading google.golang.org/genproto/googleapis/rpc v0.0.0-20240515191416-fc5f0ca64291 go: downloading golang.org/x/net v0.33.0 go: downloading golang.org/x/text v0.21.0 + nc -z localhost 6650 + echo 'Waiting for pulsar namespace to be ready...' Waiting for pulsar namespace to be ready... + i=0 + /usr/local/pulsar/bin/pulsar-admin namespaces list public check diff failed 2-th time, retry later TEST FAILED: OUTPUT DOES NOT CONTAIN 'id: 1' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ check data failed 2-th time, retry later table processor_resolved_ts_fallback.t3 exists cdc cli processor list --server http://127.0.0.1:8301 |jq '.|length'|grep -E '^2$' 2 run task successfully killing cdc1: 2876 cdc cli capture list --server http://127.0.0.1:8302 |jq '.|length'|grep -E '^1$' Error: Get "http://127.0.0.1:8301/api/v2/captures": dial tcp 127.0.0.1:8301: connect: connection refused run task failed 1-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8302 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8302 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 07:34:23 GMT < Content-Length: 1275 < Content-Type: text/plain; charset=utf-8 < { [1275 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/64b95cdc-8be7-4ffe-92d2-9beabee4aca6 {"id":"64b95cdc-8be7-4ffe-92d2-9beabee4aca6","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752057} /tidb/cdc/default/__cdc_meta__/capture/9e709ee8-80c5-47e4-b26e-0a9f389422a8 {"id":"9e709ee8-80c5-47e4-b26e-0a9f389422a8","address":"127.0.0.1:8302","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752060} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffb8935038 64b95cdc-8be7-4ffe-92d2-9beabee4aca6 /tidb/cdc/default/__cdc_meta__/owner/223195ffb893506d 9e709ee8-80c5-47e4-b26e-0a9f389422a8 /tidb/cdc/default/default/upstream/7489357992790424245 {"id":7489357992790424245,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/64b95cdc-8be7-4ffe-92d2-9beabee4aca6 {"id":"64b95cdc-8be7-4ffe-92d2-9beabee4aca6","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752057} /tidb/cdc/default/__cdc_meta__/capture/9e709ee8-80c5-47e4-b26e-0a9f389422a8 {"id":"9e709ee8-80c5-47e4-b26e-0a9f389422a8","address":"127.0.0.1:8302","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752060} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffb8935038 64b95cdc-8be7-4ffe-92d2-9beabee4aca6 /tidb/cdc/default/__cdc_meta__/owner/223195ffb893506d 9e709ee8-80c5-47e4-b26e-0a9f389422a8 /tidb/cdc/default/default/upstream/7489357992790424245 {"id":7489357992790424245,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/64b95cdc-8be7-4ffe-92d2-9beabee4aca6 {"id":"64b95cdc-8be7-4ffe-92d2-9beabee4aca6","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752057} /tidb/cdc/default/__cdc_meta__/capture/9e709ee8-80c5-47e4-b26e-0a9f389422a8 {"id":"9e709ee8-80c5-47e4-b26e-0a9f389422a8","address":"127.0.0.1:8302","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752060} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffb8935038 64b95cdc-8be7-4ffe-92d2-9beabee4aca6 /tidb/cdc/default/__cdc_meta__/owner/223195ffb893506d 9e709ee8-80c5-47e4-b26e-0a9f389422a8 /tidb/cdc/default/default/upstream/7489357992790424245 {"id":7489357992790424245,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Fri Apr 4 15:34:23 CST 2025] <<<<<< START cdc server in multi_capture case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8303/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_capture.1010210104.out server --log-file /tmp/tidb_cdc_test/multi_capture/cdc3.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_capture/cdc_data3 --cluster-id default --addr 127.0.0.1:8303 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8303/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8303 failed: Connection refused * Failed to connect to 127.0.0.1 port 8303: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table batch_add_table.finish_mark not exists for 4-th check, retry later table ddl_manager.finish_mark not exists for 7-th check, retry later + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.partition_table.cli.12789.out cli tso query --pd=http://127.0.0.1:2379 table test.finish_mark not exists for 5-th check, retry later check diff failed 1-th time, retry later go: downloading github.com/pingcap/tidb/pkg/parser v0.0.0-20241223052309-3735ed55a394 go: downloading github.com/opentracing/basictracer-go v1.1.0 go: downloading github.com/dgraph-io/ristretto v0.1.1 go: downloading github.com/json-iterator/go v1.1.12 go: downloading github.com/gorilla/mux v1.8.1 go: downloading github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 go: downloading github.com/coocood/freecache v1.2.1 go: downloading gopkg.in/yaml.v2 v2.4.0 go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible go: downloading github.com/danjacques/gofslock v0.0.0-20240212154529-d899e02bfe22 go: downloading github.com/go-sql-driver/mysql v1.7.1 go: downloading github.com/spf13/pflag v1.0.5 go: downloading github.com/influxdata/tdigest v0.0.1 go: downloading github.com/yangkeao/ldap/v3 v3.4.5-0.20230421065457-369a3bab1117 go: downloading github.com/pingcap/tipb v0.0.0-20241105053214-f91fdb81a69e go: downloading github.com/pingcap/sysutil v1.0.1-0.20240311050922-ae81ee01f3a5 go: downloading github.com/bits-and-blooms/bitset v1.14.3 go: downloading github.com/shirou/gopsutil/v3 v3.24.5 go: downloading github.com/dolthub/swiss v0.2.1 go: downloading cloud.google.com/go/storage v1.39.1 go: downloading golang.org/x/tools v0.28.0 go: downloading github.com/stretchr/testify v1.10.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v1.16.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0 go: downloading github.com/tidwall/btree v1.7.0 go: downloading github.com/otiai10/copy v1.2.0 go: downloading github.com/qri-io/jsonschema v0.2.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 go: downloading github.com/golang/snappy v0.0.4 go: downloading go.uber.org/mock v0.4.0 go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 go: downloading github.com/aws/aws-sdk-go v1.55.5 go: downloading github.com/cockroachdb/pebble v1.1.0 go: downloading github.com/klauspost/compress v1.17.9 go: downloading github.com/jfcg/sorty/v2 v2.1.0 go: downloading github.com/go-resty/resty/v2 v2.11.0 go: downloading github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df go: downloading github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 go: downloading github.com/ks3sdklib/aws-sdk-go v1.2.9 go: downloading golang.org/x/time v0.7.0 go: downloading github.com/cheggaaa/pb/v3 v3.0.8 go: downloading github.com/spkg/bom v1.0.0 go: downloading golang.org/x/term v0.27.0 go: downloading github.com/xitongsys/parquet-go v1.6.3-0.20240520233950-75e935fc3e17 go: downloading golang.org/x/oauth2 v0.24.0 check diff successfully wait process cdc.test exit for 1-th time... go: downloading github.com/fatih/color v1.18.0 go: downloading google.golang.org/api v0.170.0 go: downloading github.com/vbauerster/mpb/v7 v7.5.3 go: downloading github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef go: downloading github.com/lestrrat-go/jwx/v2 v2.0.21 go: downloading github.com/carlmjohnson/flagext v0.21.0 go: downloading github.com/jellydator/ttlcache/v3 v3.0.1 go: downloading github.com/spf13/cobra v1.8.1 go: downloading github.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0 go: downloading github.com/dolthub/maphash v0.1.0 go: downloading github.com/jedib0t/go-pretty/v6 v6.2.2 go: downloading github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 go: downloading github.com/go-asn1-ber/asn1-ber v1.5.4 go: downloading github.com/pingcap/badger v1.5.1-0.20241015064302-38533b6cbf8d go: downloading github.com/jfcg/sixb v1.3.8 go: downloading cloud.google.com/go v0.112.2 go: downloading github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 go: downloading github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 go: downloading github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc go: downloading github.com/VividCortex/ewma v1.2.0 go: downloading github.com/mattn/go-colorable v0.1.13 go: downloading github.com/mattn/go-isatty v0.0.20 go: downloading github.com/mattn/go-runewidth v0.0.16 go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 go: downloading github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 go: downloading golang.org/x/crypto v0.31.0 go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading github.com/google/pprof v0.0.0-20241001023024-f4c0cfd0cf1d go: downloading github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d go: downloading github.com/Masterminds/semver v1.5.0 go: downloading k8s.io/api v0.29.11 go: downloading github.com/emirpasic/gods v1.18.1 go: downloading github.com/robfig/cron/v3 v3.0.1 go: downloading github.com/apache/thrift v0.16.0 go: downloading github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f go: downloading github.com/qri-io/jsonpointer v0.1.1 go: downloading cloud.google.com/go/compute/metadata v0.3.0 go: downloading github.com/robfig/cron v1.2.0 go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible go: downloading github.com/tklauser/go-sysconf v0.3.12 go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading github.com/rivo/uniseg v0.4.7 go: downloading github.com/golang/glog v1.2.0 go: downloading github.com/dustin/go-humanize v1.0.1 go: downloading github.com/apache/arrow/go/v12 v12.0.1 go: downloading github.com/goccy/go-reflect v1.2.0 go: downloading cloud.google.com/go/kms v1.15.8 go: downloading github.com/pierrec/lz4/v4 v4.1.18 go: downloading github.com/lestrrat-go/blackmagic v1.0.2 go: downloading github.com/lestrrat-go/httprc v1.0.5 go: downloading github.com/lestrrat-go/iter v1.0.2 go: downloading github.com/lestrrat-go/option v1.0.1 go: downloading github.com/kylelemons/godebug v1.1.0 go: downloading github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c go: downloading github.com/golang-jwt/jwt/v5 v5.2.1 go: downloading cloud.google.com/go/iam v1.1.7 go: downloading github.com/googleapis/gax-go/v2 v2.12.3 go: downloading github.com/lestrrat-go/httpcc v1.0.1 go: downloading go.opencensus.io v0.23.1-0.20220331163232-052120675fac go: downloading go.opentelemetry.io/otel v1.24.0 go: downloading go.opentelemetry.io/otel/trace v1.24.0 go: downloading github.com/tklauser/numcpus v0.6.1 go: downloading github.com/ncw/directio v1.0.5 go: downloading github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 go: downloading github.com/golang-jwt/jwt v3.2.2+incompatible go: downloading github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 go: downloading github.com/klauspost/cpuid v1.3.1 public/default go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da go: downloading github.com/go-logr/logr v1.4.1 go: downloading go.opentelemetry.io/otel/metric v1.24.0 go: downloading github.com/go-logr/stdr v1.2.2 check diff failed 3-th time, retry later ++ date + echo '[Fri Apr 4 15:34:25 CST 2025] <<<<<< pulsar is ready >>>>>>' [Fri Apr 4 15:34:25 CST 2025] <<<<<< pulsar is ready >>>>>> + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.6684.out cli changefeed create --start-ts=457114138209157121 '--sink-uri=pulsar://127.0.0.1:6650/ticdc-sink-retry-test-4567?protocol=canal-json&enable-tidb-extension=true' -c changefeed-error TEST FAILED: OUTPUT DOES NOT CONTAIN 'id: 1' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ check data failed 3-th time, retry later go: downloading github.com/cockroachdb/redact v1.1.5 go: downloading github.com/cockroachdb/errors v1.11.1 go: downloading github.com/DataDog/zstd v1.5.5 go: downloading github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 go: downloading github.com/getsentry/sentry-go v0.27.0 go: downloading github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b go: downloading github.com/kr/pretty v0.3.1 go: downloading github.com/rogpeppe/go-internal v1.13.1 go: downloading github.com/kr/text v0.2.0 wait process cdc.test exit for 2-th time... check diff failed 1-th time, retry later + set +x + tso='457114141033496577 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 457114141033496577 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 4 15:34:25 CST 2025] <<<<<< START cdc server in partition_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.partition_table.1282912831.out server --log-file /tmp/tidb_cdc_test/partition_table/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/partition_table/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table batch_add_table.finish_mark not exists for 5-th check, retry later go: downloading golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 go: downloading github.com/google/flatbuffers v2.0.8+incompatible go: downloading github.com/goccy/go-json v0.10.2 go: downloading github.com/zeebo/xxh3 v1.0.2 go: downloading github.com/andybalholm/brotli v1.0.5 go: downloading github.com/klauspost/cpuid/v2 v2.2.4 go: downloading k8s.io/apimachinery v0.29.11 table ddl_manager.finish_mark not exists for 8-th check, retry later go: downloading gopkg.in/inf.v0 v0.9.1 go: downloading k8s.io/klog/v2 v2.120.1 go: downloading github.com/google/gofuzz v1.2.0 go: downloading sigs.k8s.io/structured-merge-diff/v4 v4.4.1 go: downloading sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd go: downloading k8s.io/utils v0.0.0-20230726121419-3b25d923346b cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 4 15:34:25 CST 2025] <<<<<< run test case capture_session_done_during_task success! >>>>>> cdc cli capture list --server http://127.0.0.1:8302 |jq '.|length'|grep -E '^1$' Error: Get "http://127.0.0.1:8301/api/v2/captures": dial tcp 127.0.0.1:8301: connect: connection refused run task failed 2-th time, retry later Create changefeed successfully! ID: changefeed-error Info: {"upstream_id":7489358005347271568,"namespace":"default","id":"changefeed-error","sink_uri":"pulsar://127.0.0.1:6650/ticdc-sink-retry-test-4567?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2025-04-04T15:34:25.949219451+08:00","start_ts":457114138209157121,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"pulsar_config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-10-ga306f4894","resolved_ts":457114138209157121,"checkpoint_ts":457114138209157121,"checkpoint_time":"2025-04-04 15:34:13.105"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... table test.finish_mark not exists for 6-th check, retry later check diff failed 2-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8303/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8303 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8303 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 07:34:26 GMT < Content-Length: 1756 < Content-Type: text/plain; charset=utf-8 < { [1756 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0218b23e-feb0-440b-bccf-694885196df1 {"id":"0218b23e-feb0-440b-bccf-694885196df1","address":"127.0.0.1:8303","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752063} /tidb/cdc/default/__cdc_meta__/capture/64b95cdc-8be7-4ffe-92d2-9beabee4aca6 {"id":"64b95cdc-8be7-4ffe-92d2-9beabee4aca6","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752057} /tidb/cdc/default/__cdc_meta__/capture/9e709ee8-80c5-47e4-b26e-0a9f389422a8 {"id":"9e709ee8-80c5-47e4-b26e-0a9f389422a8","address":"127.0.0.1:8302","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752060} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffb8935038 64b95cdc-8be7-4ffe-92d2-9beabee4aca6 /tidb/cdc/default/__cdc_meta__/owner/223195ffb893506d 9e709ee8-80c5-47e4-b26e-0a9f389422a8 /tidb/cdc/default/__cdc_meta__/owner/223195ffb8935091 0218b23e-feb0-440b-bccf-694885196df1 /tidb/cdc/default/default/upstream/7489357992790424245 {"id":7489357992790424245,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0218b23e-feb0-440b-bccf-694885196df1 {"id":"0218b23e-feb0-440b-bccf-694885196df1","address":"127.0.0.1:8303","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752063} /tidb/cdc/default/__cdc_meta__/capture/64b95cdc-8be7-4ffe-92d2-9beabee4aca6 {"id":"64b95cdc-8be7-4ffe-92d2-9beabee4aca6","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752057} /tidb/cdc/default/__cdc_meta__/capture/9e709ee8-80c5-47e4-b26e-0a9f389422a8 {"id":"9e709ee8-80c5-47e4-b26e-0a9f389422a8","address":"127.0.0.1:8302","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752060} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffb8935038 64b95cdc-8be7-4ffe-92d2-9beabee4aca6 /tidb/cdc/default/__cdc_meta__/owner/223195ffb893506d 9e709ee8-80c5-47e4-b26e-0a9f389422a8 /tidb/cdc/default/__cdc_meta__/owner/223195ffb8935091 0218b23e-feb0-440b-bccf-694885196df1 /tidb/cdc/default/default/upstream/7489357992790424245 {"id":7489357992790424245,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0218b23e-feb0-440b-bccf-694885196df1 {"id":"0218b23e-feb0-440b-bccf-694885196df1","address":"127.0.0.1:8303","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752063} /tidb/cdc/default/__cdc_meta__/capture/64b95cdc-8be7-4ffe-92d2-9beabee4aca6 {"id":"64b95cdc-8be7-4ffe-92d2-9beabee4aca6","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752057} /tidb/cdc/default/__cdc_meta__/capture/9e709ee8-80c5-47e4-b26e-0a9f389422a8 {"id":"9e709ee8-80c5-47e4-b26e-0a9f389422a8","address":"127.0.0.1:8302","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752060} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffb8935038 64b95cdc-8be7-4ffe-92d2-9beabee4aca6 /tidb/cdc/default/__cdc_meta__/owner/223195ffb893506d 9e709ee8-80c5-47e4-b26e-0a9f389422a8 /tidb/cdc/default/__cdc_meta__/owner/223195ffb8935091 0218b23e-feb0-440b-bccf-694885196df1 /tidb/cdc/default/default/upstream/7489357992790424245 {"id":7489357992790424245,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x ++ date + echo '[Fri Apr 4 15:34:26 CST 2025] <<<<<< START pulsar cluster in multi_capture case >>>>>>' [Fri Apr 4 15:34:26 CST 2025] <<<<<< START pulsar cluster in multi_capture case >>>>>> + workdir=/tmp/tidb_cdc_test/multi_capture + cluster_type=normal + cd /tmp/tidb_cdc_test/multi_capture + DEFAULT_PULSAR_HOME=/usr/local/pulsar + pulsar_dir=/usr/local/pulsar ++ cat + mtls_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderTls brokerClientTlsEnabled=true brokerClientTrustCertsFilePath=/tmp/tidb_cdc_test/multi_capture/ca.cert.pem brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls brokerClientAuthenticationParameters={"tlsCertFile":"/tmp/tidb_cdc_test/multi_capture/broker_client.cert.pem","tlsKeyFile":"/tmp/tidb_cdc_test/multi_capture/broker_client.key-pk8.pem"} brokerServicePortTls=6651 webServicePortTls=8443 tlsTrustCertsFilePath=/tmp/tidb_cdc_test/multi_capture/ca.cert.pem tlsCertificateFilePath=/tmp/tidb_cdc_test/multi_capture/server.cert.pem tlsKeyFilePath=/tmp/tidb_cdc_test/multi_capture/server.key-pk8.pem tlsRequireTrustedClientCertOnConnect=true tlsAllowInsecureConnection=false tlsCertRefreshCheckDurationSec=300' ++ cat + normal_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' ++ cat + mtls_client_conf=' webServiceUrl=https://localhost:8443/ brokerServiceUrl=pulsar+ssl://localhost:6651/ authPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls authParams=tlsCertFile:/tmp/tidb_cdc_test/multi_capture/broker_client.cert.pem,tlsKeyFile:/tmp/tidb_cdc_test/multi_capture/broker_client.key-pk8.pem tlsTrustCertsFilePath=/tmp/tidb_cdc_test/multi_capture/ca.cert.pem' ++ cat + oauth_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/ authPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 authParams={"privateKey":"/tmp/tidb_cdc_test/multi_capture/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"}' ++ cat + oauth_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderToken brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 brokerClientAuthenticationParameters={"privateKey":"file:///tmp/tidb_cdc_test/multi_capture/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"} tokenSecretKey=data:;base64,U0poWDM2X0thcFlTeWJCdEpxMzVseFhfQnJyNExSVVJTa203UW1YSkdteThwVUZXOUVJT2NWUVBzeWt6OS1qag==' ++ cat + credential_json=' { "client_id":"1234", "client_secret":"e0KVlA2EiBfjoN13olyZd2kv1KL", "audience":"cdc-api-uri", "issuer_url":"http://localhost:9096", "type": "client_credentials" }' ++ cat + cert_server_conf='[ req ] default_bits = 2048 prompt = no default_md = sha256 distinguished_name = dn [ v3_ext ] authorityKeyIdentifier=keyid,issuer:always basicConstraints=CA:FALSE keyUsage=critical, digitalSignature, keyEncipherment extendedKeyUsage=serverAuth subjectAltName=@alt_names [ dn ] CN = server [ alt_names ] DNS.1 = localhost IP.1 = 127.0.0.1' + echo ' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' + cp /usr/local/pulsar/conf/standalone.conf /tmp/tidb_cdc_test/multi_capture/pulsar_standalone.conf + pulsar_port=6650 + '[' normal == mtls ']' + '[' normal == oauth ']' + echo 'no cluster type specified, using default configuration.' no cluster type specified, using default configuration. ++ date + echo '[Fri Apr 4 15:34:26 CST 2025] <<<<<< START pulsar cluster in normal mode in multi_capture case >>>>>>' [Fri Apr 4 15:34:26 CST 2025] <<<<<< START pulsar cluster in normal mode in multi_capture case >>>>>> + echo 'Waiting for pulsar port to be ready...' Waiting for pulsar port to be ready... + i=0 + nc -z localhost 6650 + /usr/local/pulsar/bin/pulsar standalone --config /tmp/tidb_cdc_test/multi_capture/pulsar_standalone.conf -nfw --metadata-dir /tmp/tidb_cdc_test/multi_capture/pulsar-metadata --bookkeeper-dir /tmp/tidb_cdc_test/multi_capture/pulsar-bookie + i=1 + '[' 1 -gt 20 ']' + sleep 2 go: downloading github.com/pierrec/lz4 v2.6.1+incompatible check diff failed 4-th time, retry later check data successfully check diff failed 2-th time, retry later table batch_add_table.finish_mark not exists for 6-th check, retry later + set +x [Fri Apr 4 15:34:27 CST 2025] <<<<<< START Pulsar consumer in changefeed_error case >>>>>> check_changefeed_state http://127.0.0.1:2379 changefeed-error failed [CDC:ErrStartTsBeforeGC] + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-error + expected_state=failed + error_msg='[CDC:ErrStartTsBeforeGC]' + tls_dir='[CDC:ErrStartTsBeforeGC]' + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-error -s table ddl_manager.finish_mark not exists for 9-th check, retry later + info='{ "upstream_id": 7489358005347271568, "namespace": "default", "id": "changefeed-error", "state": "failed", "checkpoint_tso": 457114138209157121, "checkpoint_time": "2025-04-04 15:34:13.105", "error": { "time": "2025-04-04T15:34:26.049579927+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrStartTsBeforeGC", "message": "[CDC:ErrStartTsBeforeGC]fail to create or maintain changefeed because start-ts 457114138209156821 is earlier than or equal to GC safepoint at 457114138209157121" } }' + echo '{ "upstream_id": 7489358005347271568, "namespace": "default", "id": "changefeed-error", "state": "failed", "checkpoint_tso": 457114138209157121, "checkpoint_time": "2025-04-04 15:34:13.105", "error": { "time": "2025-04-04T15:34:26.049579927+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrStartTsBeforeGC", "message": "[CDC:ErrStartTsBeforeGC]fail to create or maintain changefeed because start-ts 457114138209156821 is earlier than or equal to GC safepoint at 457114138209157121" } }' { "upstream_id": 7489358005347271568, "namespace": "default", "id": "changefeed-error", "state": "failed", "checkpoint_tso": 457114138209157121, "checkpoint_time": "2025-04-04 15:34:13.105", "error": { "time": "2025-04-04T15:34:26.049579927+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrStartTsBeforeGC", "message": "[CDC:ErrStartTsBeforeGC]fail to create or maintain changefeed because start-ts 457114138209156821 is earlier than or equal to GC safepoint at 457114138209157121" } } ++ jq -r .state ++ echo '{' '"upstream_id":' 7489358005347271568, '"namespace":' '"default",' '"id":' '"changefeed-error",' '"state":' '"failed",' '"checkpoint_tso":' 457114138209157121, '"checkpoint_time":' '"2025-04-04' '15:34:13.105",' '"error":' '{' '"time":' '"2025-04-04T15:34:26.049579927+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrStartTsBeforeGC",' '"message":' '"[CDC:ErrStartTsBeforeGC]fail' to create or maintain changefeed because start-ts 457114138209156821 is earlier than or equal to GC safepoint at '457114138209157121"' '}' '}' + state=failed + [[ ! failed == \f\a\i\l\e\d ]] ++ jq -r .error.message ++ echo '{' '"upstream_id":' 7489358005347271568, '"namespace":' '"default",' '"id":' '"changefeed-error",' '"state":' '"failed",' '"checkpoint_tso":' 457114138209157121, '"checkpoint_time":' '"2025-04-04' '15:34:13.105",' '"error":' '{' '"time":' '"2025-04-04T15:34:26.049579927+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrStartTsBeforeGC",' '"message":' '"[CDC:ErrStartTsBeforeGC]fail' to create or maintain changefeed because start-ts 457114138209156821 is earlier than or equal to GC safepoint at '457114138209157121"' '}' '}' + message='[CDC:ErrStartTsBeforeGC]fail to create or maintain changefeed because start-ts 457114138209156821 is earlier than or equal to GC safepoint at 457114138209157121' + [[ ! [CDC:ErrStartTsBeforeGC]fail to create or maintain changefeed because start-ts 457114138209156821 is earlier than or equal to GC safepoint at 457114138209157121 =~ \[CDC:ErrStartTsBeforeGC] ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.6802.out cli changefeed resume -c changefeed-error wait process cdc.test exit for 1-th time... PASS coverage: 2.1% of statements in github.com/pingcap/tiflow/... wait process cdc.test exit for 2-th time... check diff successfully wait process cdc.test exit for 1-th time... + nc -z localhost 6650 + i=2 + '[' 2 -gt 20 ']' + sleep 2 go: downloading github.com/google/s2a-go v0.1.7 go: downloading go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 go: downloading github.com/googleapis/enterprise-certificate-proxy v0.3.2 go: downloading github.com/felixge/httpsnoop v1.0.4 table test.finish_mark not exists for 7-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 07:34:28 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [817 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/525e460d-8c40-49c9-89d2-69ff0e5336a7 {"id":"525e460d-8c40-49c9-89d2-69ff0e5336a7","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752065} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffb8c6fad5 525e460d-8c40-49c9-89d2-69ff0e5336a7 /tidb/cdc/default/default/upstream/7489358054879327813 {"id":7489358054879327813,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/525e460d-8c40-49c9-89d2-69ff0e5336a7 {"id":"525e460d-8c40-49c9-89d2-69ff0e5336a7","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752065} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffb8c6fad5 525e460d-8c40-49c9-89d2-69ff0e5336a7 /tidb/cdc/default/default/upstream/7489358054879327813 {"id":7489358054879327813,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/525e460d-8c40-49c9-89d2-69ff0e5336a7 {"id":"525e460d-8c40-49c9-89d2-69ff0e5336a7","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752065} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffb8c6fad5 525e460d-8c40-49c9-89d2-69ff0e5336a7 /tidb/cdc/default/default/upstream/7489358054879327813 {"id":7489358054879327813,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x ++ date + echo '[Fri Apr 4 15:34:28 CST 2025] <<<<<< START pulsar cluster in partition_table case >>>>>>' [Fri Apr 4 15:34:28 CST 2025] <<<<<< START pulsar cluster in partition_table case >>>>>> + workdir=/tmp/tidb_cdc_test/partition_table + cluster_type=normal + cd /tmp/tidb_cdc_test/partition_table + DEFAULT_PULSAR_HOME=/usr/local/pulsar + pulsar_dir=/usr/local/pulsar ++ cat + mtls_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderTls brokerClientTlsEnabled=true brokerClientTrustCertsFilePath=/tmp/tidb_cdc_test/partition_table/ca.cert.pem brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls brokerClientAuthenticationParameters={"tlsCertFile":"/tmp/tidb_cdc_test/partition_table/broker_client.cert.pem","tlsKeyFile":"/tmp/tidb_cdc_test/partition_table/broker_client.key-pk8.pem"} brokerServicePortTls=6651 webServicePortTls=8443 tlsTrustCertsFilePath=/tmp/tidb_cdc_test/partition_table/ca.cert.pem tlsCertificateFilePath=/tmp/tidb_cdc_test/partition_table/server.cert.pem tlsKeyFilePath=/tmp/tidb_cdc_test/partition_table/server.key-pk8.pem tlsRequireTrustedClientCertOnConnect=true tlsAllowInsecureConnection=false tlsCertRefreshCheckDurationSec=300' ++ cat + normal_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' ++ cat + mtls_client_conf=' webServiceUrl=https://localhost:8443/ brokerServiceUrl=pulsar+ssl://localhost:6651/ authPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls authParams=tlsCertFile:/tmp/tidb_cdc_test/partition_table/broker_client.cert.pem,tlsKeyFile:/tmp/tidb_cdc_test/partition_table/broker_client.key-pk8.pem tlsTrustCertsFilePath=/tmp/tidb_cdc_test/partition_table/ca.cert.pem' ++ cat + oauth_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/ authPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 authParams={"privateKey":"/tmp/tidb_cdc_test/partition_table/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"}' ++ cat + oauth_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderToken brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 brokerClientAuthenticationParameters={"privateKey":"file:///tmp/tidb_cdc_test/partition_table/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"} tokenSecretKey=data:;base64,U0poWDM2X0thcFlTeWJCdEpxMzVseFhfQnJyNExSVVJTa203UW1YSkdteThwVUZXOUVJT2NWUVBzeWt6OS1qag==' ++ cat + credential_json=' { "client_id":"1234", "client_secret":"e0KVlA2EiBfjoN13olyZd2kv1KL", "audience":"cdc-api-uri", "issuer_url":"http://localhost:9096", "type": "client_credentials" }' ++ cat + cert_server_conf='[ req ] default_bits = 2048 prompt = no default_md = sha256 distinguished_name = dn [ v3_ext ] authorityKeyIdentifier=keyid,issuer:always basicConstraints=CA:FALSE keyUsage=critical, digitalSignature, keyEncipherment extendedKeyUsage=serverAuth subjectAltName=@alt_names [ dn ] CN = server [ alt_names ] DNS.1 = localhost IP.1 = 127.0.0.1' + echo ' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' + cp /usr/local/pulsar/conf/standalone.conf /tmp/tidb_cdc_test/partition_table/pulsar_standalone.conf + pulsar_port=6650 + '[' normal == mtls ']' + '[' normal == oauth ']' + echo 'no cluster type specified, using default configuration.' no cluster type specified, using default configuration. ++ date + echo '[Fri Apr 4 15:34:28 CST 2025] <<<<<< START pulsar cluster in normal mode in partition_table case >>>>>>' [Fri Apr 4 15:34:28 CST 2025] <<<<<< START pulsar cluster in normal mode in partition_table case >>>>>> + echo 'Waiting for pulsar port to be ready...' Waiting for pulsar port to be ready... + i=0 + /usr/local/pulsar/bin/pulsar standalone --config /tmp/tidb_cdc_test/partition_table/pulsar_standalone.conf -nfw --metadata-dir /tmp/tidb_cdc_test/partition_table/pulsar-metadata --bookkeeper-dir /tmp/tidb_cdc_test/partition_table/pulsar-bookie + nc -z localhost 6650 + i=1 + '[' 1 -gt 20 ']' + sleep 2 cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 4 15:34:28 CST 2025] <<<<<< run test case simple success! >>>>>> go: downloading github.com/jmespath/go-jmespath v0.4.0 wait process cdc.test exit for 2-th time... check diff successfully % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 check diff failed 5-th time, retry later + set +x table changefeed_error.usertable not exists for 1-th check, retry later cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 4 15:34:29 CST 2025] <<<<<< run test case split_region success! >>>>>> table batch_add_table.finish_mark not exists for 7-th check, retry later table ddl_manager.finish_mark not exists for 10-th check, retry later cdc cli capture list --server http://127.0.0.1:8302 |jq '.|length'|grep -E '^1$' Error: Get "http://127.0.0.1:8301/api/v2/captures": dial tcp 127.0.0.1:8301: connect: connection refused run task failed 3-th time, retry later + nc -z localhost 6650 + echo 'Waiting for pulsar namespace to be ready...' Waiting for pulsar namespace to be ready... + i=0 + /usr/local/pulsar/bin/pulsar-admin namespaces list public table test.finish_mark not exists for 8-th check, retry later + nc -z localhost 6650 + i=2 + '[' 2 -gt 20 ']' + sleep 2 check diff successfully table batch_add_table.finish_mark not exists for 8-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/region_merge/run.sh using Sink-Type: pulsar... <<================= The 1 times to try to start tidb cluster... table ddl_manager.finish_mark not exists for 11-th check, retry later table changefeed_error.usertable exists check diff failed 1-th time, retry later check diff failed 1-th time, retry later public/default table test.finish_mark not exists for 9-th check, retry later + nc -z localhost 6650 + echo 'Waiting for pulsar namespace to be ready...' Waiting for pulsar namespace to be ready... + i=0 + /usr/local/pulsar/bin/pulsar-admin namespaces list public ++ date + echo '[Fri Apr 4 15:34:32 CST 2025] <<<<<< pulsar is ready >>>>>>' [Fri Apr 4 15:34:32 CST 2025] <<<<<< pulsar is ready >>>>>> + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_capture.cli.10564.out cli changefeed create --start-ts=457114137710034945 '--sink-uri=pulsar://127.0.0.1:6650/ticdc-multi-capture-test-32510?protocol=canal-json&enable-tidb-extension=true' --server=127.0.0.1:8301 Create changefeed successfully! ID: 7b1c1475-ad53-4732-826a-45fc7952d2a0 Info: {"upstream_id":7489357992790424245,"namespace":"default","id":"7b1c1475-ad53-4732-826a-45fc7952d2a0","sink_uri":"pulsar://127.0.0.1:6650/ticdc-multi-capture-test-32510?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2025-04-04T15:34:33.408498091+08:00","start_ts":457114137710034945,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"pulsar_config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-10-ga306f4894","resolved_ts":457114137710034945,"checkpoint_ts":457114137710034945,"checkpoint_time":"2025-04-04 15:34:11.201"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... table batch_add_table.finish_mark not exists for 9-th check, retry later table ddl_manager.finish_mark not exists for 12-th check, retry later check diff successfully ***************** properties ***************** "mysql.db"="changefeed_error" "mysql.port"="4000" "operationcount"="0" "threadcount"="4" "workload"="core" "updateproportion"="0" "dotransactions"="false" "readallfields"="true" "readproportion"="0" "mysql.user"="root" "mysql.host"="127.0.0.1" "requestdistribution"="uniform" "insertproportion"="0" "scanproportion"="0" "recordcount"="20" ********************************************** Run finished, takes 4.900416ms INSERT - Takes(s): 0.0, Count: 20, OPS: 5936.0, Avg(us): 836, Min(us): 436, Max(us): 1886, 95th(us): 2000, 99th(us): 2000 check diff failed 2-th time, retry later check diff successfully {"id":"51c50a27-0a75-49c0-be9b-159863c14d2e","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752055} check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/capture' 'capture' + key_prefix=/tidb/cdc/default/__cdc_meta__/capture + message=capture ++ etcdctl get /tidb/cdc/default/__cdc_meta__/capture --prefix --keys-only + info=/tidb/cdc/default/__cdc_meta__/capture/51c50a27-0a75-49c0-be9b-159863c14d2e + [[ /tidb/cdc/default/__cdc_meta__/capture/51c50a27-0a75-49c0-be9b-159863c14d2e =~ capture ]] + echo 'capture contains in etcd /tidb/cdc/default/__cdc_meta__/capture/51c50a27-0a75-49c0-be9b-159863c14d2e' capture contains in etcd /tidb/cdc/default/__cdc_meta__/capture/51c50a27-0a75-49c0-be9b-159863c14d2e + echo 'check failed' check failed + exit 1 run task failed 1-th time, retry later table test.finish_mark not exists for 10-th check, retry later public/default + set +x [Fri Apr 4 15:34:34 CST 2025] <<<<<< START Pulsar consumer in multi_capture case >>>>>> table multi_capture_1.usertable not exists for 1-th check, retry later ++ date + echo '[Fri Apr 4 15:34:34 CST 2025] <<<<<< pulsar is ready >>>>>>' [Fri Apr 4 15:34:34 CST 2025] <<<<<< pulsar is ready >>>>>> + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.partition_table.cli.13302.out cli changefeed create --start-ts=457114141033496577 '--sink-uri=pulsar://127.0.0.1:6650/ticdc-partition-table-test-2791?protocol=canal-json&enable-tidb-extension=true' start tidb cluster in /tmp/tidb_cdc_test/region_merge Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... table batch_add_table.finish_mark not exists for 10-th check, retry later table ddl_manager.finish_mark not exists for 13-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/ddl_attributes/run.sh using Sink-Type: pulsar... <<================= The 1 times to try to start tidb cluster... Create changefeed successfully! ID: 593d3f4c-28e1-4456-a6ea-4b81c5a92999 Info: {"upstream_id":7489358054879327813,"namespace":"default","id":"593d3f4c-28e1-4456-a6ea-4b81c5a92999","sink_uri":"pulsar://127.0.0.1:6650/ticdc-partition-table-test-2791?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2025-04-04T15:34:35.628612776+08:00","start_ts":457114141033496577,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"pulsar_config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-10-ga306f4894","resolved_ts":457114141033496577,"checkpoint_ts":457114141033496577,"checkpoint_time":"2025-04-04 15:34:23.879"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/capture' 'capture' + key_prefix=/tidb/cdc/default/__cdc_meta__/capture + message=capture ++ etcdctl get /tidb/cdc/default/__cdc_meta__/capture --prefix --keys-only + info=/tidb/cdc/default/__cdc_meta__/capture/51c50a27-0a75-49c0-be9b-159863c14d2e + [[ /tidb/cdc/default/__cdc_meta__/capture/51c50a27-0a75-49c0-be9b-159863c14d2e =~ capture ]] + echo 'capture contains in etcd /tidb/cdc/default/__cdc_meta__/capture/51c50a27-0a75-49c0-be9b-159863c14d2e' capture contains in etcd /tidb/cdc/default/__cdc_meta__/capture/51c50a27-0a75-49c0-be9b-159863c14d2e + echo 'check failed' check failed + exit 1 run task failed 2-th time, retry later table test.finish_mark not exists for 11-th check, retry later cdc cli capture list --server http://127.0.0.1:8302 |jq '.|length'|grep -E '^1$' 1 run task successfully check diff failed 1-th time, retry later check diff failed 3-th time, retry later + set +x [Fri Apr 4 15:34:37 CST 2025] <<<<<< START Pulsar consumer in partition_table case >>>>>> table multi_capture_1.usertable exists table multi_capture_2.usertable not exists for 1-th check, retry later table batch_add_table.finish_mark not exists for 11-th check, retry later table ddl_manager.finish_mark not exists for 14-th check, retry later check diff successfully check diff failed 4-th time, retry later wait process cdc.test exit for 1-th time... table test.finish_mark not exists for 12-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/ddl_attributes Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... check diff successfully table multi_capture_2.usertable exists table multi_capture_3.usertable not exists for 1-th check, retry later wait process cdc.test exit for 2-th time... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 4 15:34:39 CST 2025] <<<<<< run test case processor_resolved_ts_fallback success! >>>>>> wait process cdc.test exit for 1-th time... table batch_add_table.finish_mark not exists for 12-th check, retry later table ddl_manager.finish_mark not exists for 15-th check, retry later wait process cdc.test exit for 2-th time... check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/capture' 'capture' + key_prefix=/tidb/cdc/default/__cdc_meta__/capture + message=capture ++ etcdctl get /tidb/cdc/default/__cdc_meta__/capture --prefix --keys-only + info=/tidb/cdc/default/__cdc_meta__/capture/51c50a27-0a75-49c0-be9b-159863c14d2e + [[ /tidb/cdc/default/__cdc_meta__/capture/51c50a27-0a75-49c0-be9b-159863c14d2e =~ capture ]] + echo 'capture contains in etcd /tidb/cdc/default/__cdc_meta__/capture/51c50a27-0a75-49c0-be9b-159863c14d2e' capture contains in etcd /tidb/cdc/default/__cdc_meta__/capture/51c50a27-0a75-49c0-be9b-159863c14d2e + echo 'check failed' check failed + exit 1 run task failed 3-th time, retry later check diff failed 5-th time, retry later table test.finish_mark not exists for 13-th check, retry later cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 4 15:34:40 CST 2025] <<<<<< run test case owner_resign success! >>>>>> =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/cdc_server_tips/run.sh using Sink-Type: pulsar... <<================= The 1 times to try to start tidb cluster... Verifying downstream PD is started... table multi_capture_3.usertable exists table multi_capture_4.usertable not exists for 1-th check, retry later Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table batch_add_table.finish_mark not exists for 13-th check, retry later \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-3484/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } table ddl_manager.finish_mark not exists for 16-th check, retry later [Pipeline] // dir [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } table test.finish_mark not exists for 14-th check, retry later [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } table multi_capture_4.usertable exists check diff successfully VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489358145248701036 TiDB Cluster ID. tikv_gc_leader_uuid 657fee4d4a80014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-vgqp-h2d14, pid:4837, start at 2025-04-04 15:34:42.77782981 +0800 CST m=+1.794808194 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:36:42.786 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:34:42.780 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:24:42.780 +0800 All versions after safe point can be accessed. (DO NOT EDIT) check diff failed 1-th time, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489358145248701036 TiDB Cluster ID. tikv_gc_leader_uuid 657fee4d4a80014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-vgqp-h2d14, pid:4837, start at 2025-04-04 15:34:42.77782981 +0800 CST m=+1.794808194 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:36:42.786 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:34:42.780 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:24:42.780 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489358138090141357 TiDB Cluster ID. tikv_gc_leader_uuid 657fee4d3e8000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-vgqp-h2d14, pid:4916, start at 2025-04-04 15:34:42.71155506 +0800 CST m=+1.652601866 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:36:42.719 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:34:42.682 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:24:42.682 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/region_merge/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/region_merge/tiflash/log/error.log check diff failed 1-th time, retry later arg matches is ArgMatches { args: {"memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/region_merge/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/region_merge/tiflash/log/proxy.log"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/region_merge/tiflash-proxy.toml"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table batch_add_table.finish_mark not exists for 14-th check, retry later table partition_table.t exists table partition_table.t1 not exists for 1-th check, retry later table ddl_manager.finish_mark not exists for 17-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/cdc_server_tips Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table partition_table.t1 exists table partition_table.t2 not exists for 1-th check, retry later table test.finish_mark not exists for 15-th check, retry later [Fri Apr 4 15:34:45 CST 2025] <<<<<< START cdc server in region_merge case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.region_merge.62426244.out server --log-file /tmp/tidb_cdc_test/region_merge/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/region_merge/cdc_data --cluster-id default + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table batch_add_table.finish_mark not exists for 15-th check, retry later check diff successfully ***************** properties ***************** "mysql.db"="multi_capture_1" "scanproportion"="0" "mysql.user"="root" "recordcount"="20" "mysql.host"="127.0.0.1" "readproportion"="0" "requestdistribution"="uniform" "operationcount"="0" "updateproportion"="0" "insertproportion"="0" "mysql.port"="4000" "readallfields"="true" "dotransactions"="false" "workload"="core" "threadcount"="2" ********************************************** Run finished, takes 12.702953ms INSERT - Takes(s): 0.0, Count: 20, OPS: 1755.3, Avg(us): 1220, Min(us): 471, Max(us): 2881, 95th(us): 3000, 99th(us): 3000 ***************** properties ***************** "mysql.port"="4000" "updateproportion"="0" "insertproportion"="0" "recordcount"="20" "mysql.user"="root" "workload"="core" "requestdistribution"="uniform" "scanproportion"="0" "readproportion"="0" "mysql.host"="127.0.0.1" "threadcount"="2" "dotransactions"="false" "operationcount"="0" "mysql.db"="multi_capture_2" "readallfields"="true" ********************************************** Run finished, takes 13.343262ms INSERT - Takes(s): 0.0, Count: 20, OPS: 1630.6, Avg(us): 1282, Min(us): 451, Max(us): 2839, 95th(us): 3000, 99th(us): 3000 ***************** properties ***************** "insertproportion"="0" "workload"="core" "scanproportion"="0" "mysql.port"="4000" "mysql.host"="127.0.0.1" "mysql.user"="root" "recordcount"="20" "requestdistribution"="uniform" "dotransactions"="false" "updateproportion"="0" "threadcount"="2" "mysql.db"="multi_capture_3" "readproportion"="0" "readallfields"="true" "operationcount"="0" ********************************************** Run finished, takes 11.409435ms INSERT - Takes(s): 0.0, Count: 20, OPS: 2012.4, Avg(us): 1093, Min(us): 472, Max(us): 1807, 95th(us): 2000, 99th(us): 2000 ***************** properties ***************** "updateproportion"="0" "requestdistribution"="uniform" "threadcount"="2" "workload"="core" "scanproportion"="0" "recordcount"="20" "mysql.host"="127.0.0.1" "mysql.user"="root" "dotransactions"="false" "readproportion"="0" "operationcount"="0" "mysql.db"="multi_capture_4" "mysql.port"="4000" "insertproportion"="0" "readallfields"="true" ********************************************** Run finished, takes 10.589612ms INSERT - Takes(s): 0.0, Count: 20, OPS: 2106.4, Avg(us): 999, Min(us): 485, Max(us): 1698, 95th(us): 2000, 99th(us): 2000 check diff failed 1-th time, retry later table ddl_manager.finish_mark not exists for 18-th check, retry later Verifying downstream PD is started... check diff failed 2-th time, retry later table partition_table.t2 not exists for 2-th check, retry later check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/capture' 'capture' + key_prefix=/tidb/cdc/default/__cdc_meta__/capture + message=capture ++ etcdctl get /tidb/cdc/default/__cdc_meta__/capture --prefix --keys-only + info= + [[ '' =~ capture ]] + echo 'check pass' check pass + exit 0 run task successfully check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/owner' 'owner' + key_prefix=/tidb/cdc/default/__cdc_meta__/owner + message=owner ++ etcdctl get /tidb/cdc/default/__cdc_meta__/owner --prefix --keys-only + info= + [[ '' =~ owner ]] + echo 'check pass' check pass + exit 0 run task successfully [Fri Apr 4 15:34:46 CST 2025] <<<<<< START cdc server in changefeed_error case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/NewChangefeedRetryError=return(true)' + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.70387040.out server --log-file /tmp/tidb_cdc_test/changefeed_error/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_error/cdc_data --cluster-id default + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489358153106219116 TiDB Cluster ID. tikv_gc_leader_uuid 657fee503500007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-qj98-cgzmv, pid:4788, start at 2025-04-04 15:34:45.726458412 +0800 CST m=+1.558609846 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:36:45.734 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:34:45.716 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:24:45.716 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489358153106219116 TiDB Cluster ID. tikv_gc_leader_uuid 657fee503500007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-qj98-cgzmv, pid:4788, start at 2025-04-04 15:34:45.726458412 +0800 CST m=+1.558609846 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:36:45.734 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:34:45.716 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:24:45.716 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489358163014712194 TiDB Cluster ID. tikv_gc_leader_uuid 657fee50338000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-qj98-cgzmv, pid:4864, start at 2025-04-04 15:34:45.754372703 +0800 CST m=+1.516137573 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:36:45.762 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:34:45.759 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:24:45.759 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/ddl_attributes/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/ddl_attributes/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/ddl_attributes/tiflash/db/proxy"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/ddl_attributes/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/ddl_attributes/tiflash/log/proxy.log"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table test.finish_mark not exists for 16-th check, retry later table ddl_manager.finish_mark not exists for 19-th check, retry later check diff failed 3-th time, retry later table batch_add_table.finish_mark not exists for 16-th check, retry later check diff successfully wait process cdc.test exit for 1-th time... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_attributes.cli.6289.out cli tso query --pd=http://127.0.0.1:2379 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 07:34:48 GMT < Content-Length: 861 < Content-Type: text/plain; charset=utf-8 < { [861 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ee5a8119-9b13-4c26-939b-b5b93a8886cb {"id":"ee5a8119-9b13-4c26-939b-b5b93a8886cb","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752085} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffb91e13d5 ee5a8119-9b13-4c26-939b-b5b93a8886cb /tidb/cdc/default/default/upstream/7489358145248701036 {"id":7489358145248701036,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ee5a8119-9b13-4c26-939b-b5b93a8886cb {"id":"ee5a8119-9b13-4c26-939b-b5b93a8886cb","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752085} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffb91e13d5 ee5a8119-9b13-4c26-939b-b5b93a8886cb /tidb/cdc/default/default/upstream/7489358145248701036 {"id":7489358145248701036,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ee5a8119-9b13-4c26-939b-b5b93a8886cb {"id":"ee5a8119-9b13-4c26-939b-b5b93a8886cb","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752085} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffb91e13d5 ee5a8119-9b13-4c26-939b-b5b93a8886cb /tidb/cdc/default/default/upstream/7489358145248701036 {"id":7489358145248701036,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x ++ date + echo '[Fri Apr 4 15:34:48 CST 2025] <<<<<< START pulsar cluster in region_merge case >>>>>>' [Fri Apr 4 15:34:48 CST 2025] <<<<<< START pulsar cluster in region_merge case >>>>>> + workdir=/tmp/tidb_cdc_test/region_merge + cluster_type=mtls + cd /tmp/tidb_cdc_test/region_merge + DEFAULT_PULSAR_HOME=/usr/local/pulsar + pulsar_dir=/usr/local/pulsar ++ cat + mtls_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderTls brokerClientTlsEnabled=true brokerClientTrustCertsFilePath=/tmp/tidb_cdc_test/region_merge/ca.cert.pem brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls brokerClientAuthenticationParameters={"tlsCertFile":"/tmp/tidb_cdc_test/region_merge/broker_client.cert.pem","tlsKeyFile":"/tmp/tidb_cdc_test/region_merge/broker_client.key-pk8.pem"} brokerServicePortTls=6651 webServicePortTls=8443 tlsTrustCertsFilePath=/tmp/tidb_cdc_test/region_merge/ca.cert.pem tlsCertificateFilePath=/tmp/tidb_cdc_test/region_merge/server.cert.pem tlsKeyFilePath=/tmp/tidb_cdc_test/region_merge/server.key-pk8.pem tlsRequireTrustedClientCertOnConnect=true tlsAllowInsecureConnection=false tlsCertRefreshCheckDurationSec=300' ++ cat + normal_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' ++ cat + mtls_client_conf=' webServiceUrl=https://localhost:8443/ brokerServiceUrl=pulsar+ssl://localhost:6651/ authPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls authParams=tlsCertFile:/tmp/tidb_cdc_test/region_merge/broker_client.cert.pem,tlsKeyFile:/tmp/tidb_cdc_test/region_merge/broker_client.key-pk8.pem tlsTrustCertsFilePath=/tmp/tidb_cdc_test/region_merge/ca.cert.pem' ++ cat + oauth_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/ authPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 authParams={"privateKey":"/tmp/tidb_cdc_test/region_merge/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"}' ++ cat + oauth_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderToken brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 brokerClientAuthenticationParameters={"privateKey":"file:///tmp/tidb_cdc_test/region_merge/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"} tokenSecretKey=data:;base64,U0poWDM2X0thcFlTeWJCdEpxMzVseFhfQnJyNExSVVJTa203UW1YSkdteThwVUZXOUVJT2NWUVBzeWt6OS1qag==' ++ cat + credential_json=' { "client_id":"1234", "client_secret":"e0KVlA2EiBfjoN13olyZd2kv1KL", "audience":"cdc-api-uri", "issuer_url":"http://localhost:9096", "type": "client_credentials" }' ++ cat + cert_server_conf='[ req ] default_bits = 2048 prompt = no default_md = sha256 distinguished_name = dn [ v3_ext ] authorityKeyIdentifier=keyid,issuer:always basicConstraints=CA:FALSE keyUsage=critical, digitalSignature, keyEncipherment extendedKeyUsage=serverAuth subjectAltName=@alt_names [ dn ] CN = server [ alt_names ] DNS.1 = localhost IP.1 = 127.0.0.1' + echo ' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' + cp /usr/local/pulsar/conf/standalone.conf /tmp/tidb_cdc_test/region_merge/pulsar_standalone.conf + pulsar_port=6650 + '[' mtls == mtls ']' + pulsar_port=6651 + gen_mtls_config + openssl genrsa -out ca.key.pem 2048 Generating RSA private key, 2048 bit long modulus (2 primes) ......................................................................................................................................................+++++ ...+++++ e is 65537 (0x010001) + openssl req -x509 -new -nodes -key ca.key.pem -subj /CN=CARoot -days 365 -out ca.cert.pem + openssl genrsa -out server.key.pem 2048 Generating RSA private key, 2048 bit long modulus (2 primes) ..................................................+++++ ........................................................................................+++++ e is 65537 (0x010001) + openssl pkcs8 -topk8 -inform PEM -outform PEM -in server.key.pem -out server.key-pk8.pem -nocrypt + echo '[ req ] default_bits = 2048 prompt = no default_md = sha256 distinguished_name = dn [ v3_ext ] authorityKeyIdentifier=keyid,issuer:always basicConstraints=CA:FALSE keyUsage=critical, digitalSignature, keyEncipherment extendedKeyUsage=serverAuth subjectAltName=@alt_names [ dn ] CN = server [ alt_names ] DNS.1 = localhost IP.1 = 127.0.0.1' + openssl req -new -config server.conf -key server.key.pem -out server.csr.pem -sha256 + openssl x509 -req -in server.csr.pem -CA ca.cert.pem -CAkey ca.key.pem -CAcreateserial -out server.cert.pem -days 365 -extensions v3_ext -extfile server.conf -sha256 Signature ok subject=CN = server Getting CA Private Key + openssl genrsa -out broker_client.key.pem 2048 Generating RSA private key, 2048 bit long modulus (2 primes) ............+++++ ......................................+++++ e is 65537 (0x010001) + openssl pkcs8 -topk8 -inform PEM -outform PEM -in broker_client.key.pem -out broker_client.key-pk8.pem -nocrypt + openssl req -new -subj /CN=broker_client -key broker_client.key.pem -out broker_client.csr.pem -sha256 + openssl x509 -req -in broker_client.csr.pem -CA ca.cert.pem -CAkey ca.key.pem -CAcreateserial -out broker_client.cert.pem -days 365 -sha256 Signature ok subject=CN = broker_client Getting CA Private Key + echo ' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderTls brokerClientTlsEnabled=true brokerClientTrustCertsFilePath=/tmp/tidb_cdc_test/region_merge/ca.cert.pem brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls brokerClientAuthenticationParameters={"tlsCertFile":"/tmp/tidb_cdc_test/region_merge/broker_client.cert.pem","tlsKeyFile":"/tmp/tidb_cdc_test/region_merge/broker_client.key-pk8.pem"} brokerServicePortTls=6651 webServicePortTls=8443 tlsTrustCertsFilePath=/tmp/tidb_cdc_test/region_merge/ca.cert.pem tlsCertificateFilePath=/tmp/tidb_cdc_test/region_merge/server.cert.pem tlsKeyFilePath=/tmp/tidb_cdc_test/region_merge/server.key-pk8.pem tlsRequireTrustedClientCertOnConnect=true tlsAllowInsecureConnection=false tlsCertRefreshCheckDurationSec=300' + echo ' webServiceUrl=https://localhost:8443/ brokerServiceUrl=pulsar+ssl://localhost:6651/ authPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls authParams=tlsCertFile:/tmp/tidb_cdc_test/region_merge/broker_client.cert.pem,tlsKeyFile:/tmp/tidb_cdc_test/region_merge/broker_client.key-pk8.pem tlsTrustCertsFilePath=/tmp/tidb_cdc_test/region_merge/ca.cert.pem' ++ date + echo '[Fri Apr 4 15:34:48 CST 2025] <<<<<< START pulsar cluster in mtls mode in region_merge case >>>>>>' [Fri Apr 4 15:34:48 CST 2025] <<<<<< START pulsar cluster in mtls mode in region_merge case >>>>>> + echo 'Waiting for pulsar port to be ready...' Waiting for pulsar port to be ready... + i=0 + nc -z localhost 6651 + /usr/local/pulsar/bin/pulsar standalone --config /tmp/tidb_cdc_test/region_merge/pulsar_standalone.conf -nfw --metadata-dir /tmp/tidb_cdc_test/region_merge/pulsar-metadata --bookkeeper-dir /tmp/tidb_cdc_test/region_merge/pulsar-bookie + i=1 + '[' 1 -gt 20 ']' + sleep 2 wait process cdc.test exit for 2-th time... table partition_table.t2 not exists for 3-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 07:34:49 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3520 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/60f58e50-83da-4b64-bd40-1a843417ea98 {"id":"60f58e50-83da-4b64-bd40-1a843417ea98","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752086} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 3 /tidb/cdc/default/__cdc_meta__/owner/223195ffb896d829 60f58e50-83da-4b64-bd40-1a843417ea98 /tidb/cdc/default/default/changefeed/info/changefeed-error {"upstream-id":7489358005347271568,"namespace":"default","changefeed-id":"changefeed-error","sink-uri":"pulsar://127.0.0.1:6650/ticdc-sink-retry-test-4567?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2025-04-04T15:34:25.949219451+08:00","start-ts":457114138209157121,"target-ts":0,"admin-job-type":1,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"pulsar-config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"pending","error":{"time":"2025-04-04T15:34:46.793560176+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrOwnerUnknown","message":"failpoint injected retriable error"},"warning":null,"creator-version":"v9.0.0-beta.1-10-ga306f4894","epoch":457114147030302726} /tidb/cdc/default/default/changefeed/status/changefeed-error {"checkpoint-ts":457114143530680325,"min-table-barrier-ts":457114143530680325,"admin-job-type":1} /tidb/cdc/default/default/upstream/7489358005347271568 {"id":7489358005347271568,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/60f58e50-83da-4b64-bd40-1a843417ea98 {"id":"60f58e50-83da-4b64-bd40-1a843417ea98","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752086} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 3 /tidb/cdc/default/__cdc_meta__/owner/223195ffb896d829 60f58e50-83da-4b64-bd40-1a843417ea98 /tidb/cdc/default/default/changefeed/info/changefeed-error {"upstream-id":7489358005347271568,"namespace":"default","changefeed-id":"changefeed-error","sink-uri":"pulsar://127.0.0.1:6650/ticdc-sink-retry-test-4567?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2025-04-04T15:34:25.949219451+08:00","start-ts":457114138209157121,"target-ts":0,"admin-job-type":1,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"pulsar-config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"pending","error":{"time":"2025-04-04T15:34:46.793560176+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrOwnerUnknown","message":"failpoint injected retriable error"},"warning":null,"creator-version":"v9.0.0-beta.1-10-ga306f4894","epoch":457114147030302726} /tidb/cdc/default/default/changefeed/status/changefeed-error {"checkpoint-ts":457114143530680325,"min-table-barrier-ts":457114143530680325,"admin-job-type":1} /tidb/cdc/default/default/upstream/7489358005347271568 {"id":7489358005347271568,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/60f58e50-83da-4b64-bd40-1a843417ea98 {"id":"60f58e50-83da-4b64-bd40-1a843417ea98","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752086} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 3 /tidb/cdc/default/__cdc_meta__/owner/223195ffb896d829 60f58e50-83da-4b64-bd40-1a843417ea98 /tidb/cdc/default/default/changefeed/info/changefeed-error {"upstream-id":7489358005347271568,"namespace":"default","changefeed-id":"changefeed-error","sink-uri":"pulsar://127.0.0.1:6650/ticdc-sink-retry-test-4567?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2025-04-04T15:34:25.949219451+08:00","start-ts":457114138209157121,"target-ts":0,"admin-job-type":1,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"pulsar-config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"pending","error":{"time":"2025-04-04T15:34:46.793560176+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrOwnerUnknown","message":"failpoint injected retriable error"},"warning":null,"creator-version":"v9.0.0-beta.1-10-ga306f4894","epoch":457114147030302726} /tidb/cdc/default/default/changefeed/status/changefeed-error {"checkpoint-ts":457114143530680325,"min-table-barrier-ts":457114143530680325,"admin-job-type":1} /tidb/cdc/default/default/upstream/7489358005347271568 {"id":7489358005347271568,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x check_changefeed_state http://127.0.0.1:2379 changefeed-error warning failpoint injected retriable error + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-error + expected_state=warning + error_msg=failpoint + tls_dir=error + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-error -s wait process cdc.test exit for 3-th time... table test.finish_mark not exists for 17-th check, retry later + info='{ "upstream_id": 7489358005347271568, "namespace": "default", "id": "changefeed-error", "state": "warning", "checkpoint_tso": 457114143530680325, "checkpoint_time": "2025-04-04 15:34:33.405", "error": { "time": "2025-04-04T15:34:46.793560176+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrOwnerUnknown", "message": "failpoint injected retriable error" } }' + echo '{ "upstream_id": 7489358005347271568, "namespace": "default", "id": "changefeed-error", "state": "warning", "checkpoint_tso": 457114143530680325, "checkpoint_time": "2025-04-04 15:34:33.405", "error": { "time": "2025-04-04T15:34:46.793560176+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrOwnerUnknown", "message": "failpoint injected retriable error" } }' { "upstream_id": 7489358005347271568, "namespace": "default", "id": "changefeed-error", "state": "warning", "checkpoint_tso": 457114143530680325, "checkpoint_time": "2025-04-04 15:34:33.405", "error": { "time": "2025-04-04T15:34:46.793560176+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrOwnerUnknown", "message": "failpoint injected retriable error" } } ++ echo '{' '"upstream_id":' 7489358005347271568, '"namespace":' '"default",' '"id":' '"changefeed-error",' '"state":' '"warning",' '"checkpoint_tso":' 457114143530680325, '"checkpoint_time":' '"2025-04-04' '15:34:33.405",' '"error":' '{' '"time":' '"2025-04-04T15:34:46.793560176+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrOwnerUnknown",' '"message":' '"failpoint' injected retriable 'error"' '}' '}' ++ jq -r .state + state=warning + [[ ! warning == \w\a\r\n\i\n\g ]] ++ echo '{' '"upstream_id":' 7489358005347271568, '"namespace":' '"default",' '"id":' '"changefeed-error",' '"state":' '"warning",' '"checkpoint_tso":' 457114143530680325, '"checkpoint_time":' '"2025-04-04' '15:34:33.405",' '"error":' '{' '"time":' '"2025-04-04T15:34:46.793560176+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrOwnerUnknown",' '"message":' '"failpoint' injected retriable 'error"' '}' '}' ++ jq -r .error.message + message='failpoint injected retriable error' + [[ ! failpoint injected retriable error =~ failpoint ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.7125.out cli changefeed remove -c changefeed-error table ddl_manager.finish_mark not exists for 20-th check, retry later cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Fri Apr 4 15:34:49 CST 2025] <<<<<< run test case multi_capture success! >>>>>> check diff failed 4-th time, retry later table batch_add_table.finish_mark not exists for 17-th check, retry later Changefeed remove successfully. ID: changefeed-error CheckpointTs: 457114143530680325 SinkURI: pulsar://127.0.0.1:6650/ticdc-sink-retry-test-4567?protocol=canal-json&enable-tidb-extension=true PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x + tso='457114147596795905 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 457114147596795905 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 4 15:34:50 CST 2025] <<<<<< START cdc server in ddl_attributes case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_attributes.63206322.out server --log-file /tmp/tidb_cdc_test/ddl_attributes/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_attributes/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table partition_table.t2 not exists for 4-th check, retry later table test.finish_mark not exists for 18-th check, retry later + nc -z localhost 6651 + i=2 + '[' 2 -gt 20 ']' + sleep 2 \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-3484/tiflow-cdc already exists) table batch_add_table.finish_mark exists check diff successfully + set +x check_no_changefeed 127.0.0.1:2379 parse error: Invalid numeric literal at line 1, column 6 run task successfully [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // timeout [Pipeline] } table ddl_manager.finish_mark not exists for 21-th check, retry later [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv wait process cdc.test exit for 1-th time... [Pipeline] } VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489358180080460011 TiDB Cluster ID. tikv_gc_leader_uuid 657fee56138000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-b84s-w4c6p, pid:4889, start at 2025-04-04 15:34:51.771187355 +0800 CST m=+1.573227200 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:36:51.779 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:34:51.777 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:24:51.777 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489358180080460011 TiDB Cluster ID. tikv_gc_leader_uuid 657fee56138000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-b84s-w4c6p, pid:4889, start at 2025-04-04 15:34:51.771187355 +0800 CST m=+1.573227200 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:36:51.779 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:34:51.777 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:24:51.777 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489358189623192251 TiDB Cluster ID. tikv_gc_leader_uuid 657fee56134000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-b84s-w4c6p, pid:4956, start at 2025-04-04 15:34:51.744990341 +0800 CST m=+1.484605534 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:36:51.752 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:34:51.725 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:24:51.725 +0800 All versions after safe point can be accessed. (DO NOT EDIT) wait process cdc.test exit for 1-th time... check diff failed 5-th time, retry later [Pipeline] // node [Pipeline] } Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/cdc_server_tips/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/cdc_server_tips/tiflash/log/error.log arg matches is ArgMatches { args: {"config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/cdc_server_tips/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/cdc_server_tips/tiflash/log/proxy.log"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/cdc_server_tips/tiflash/db/proxy"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } wait process cdc.test exit for 2-th time... table partition_table.t2 exists table partition_table.finish_mark not exists for 1-th check, retry later wait process cdc.test exit for 2-th time... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/processor_etcd_worker_delay/run.sh using Sink-Type: pulsar... <<================= The 1 times to try to start tidb cluster... table test.finish_mark exists check diff successfully + nc -z localhost 6651 + echo 'Waiting for pulsar namespace to be ready...' Waiting for pulsar namespace to be ready... + i=0 + /usr/local/pulsar/bin/pulsar-admin namespaces list public cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/owner' 'owner' + key_prefix=/tidb/cdc/default/__cdc_meta__/owner + message=owner ++ etcdctl get /tidb/cdc/default/__cdc_meta__/owner --prefix --keys-only + info= + [[ '' =~ owner ]] + echo 'check pass' check pass + exit 0 run task successfully [Fri Apr 4 15:34:53 CST 2025] <<<<<< START cdc server in changefeed_error case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/InjectChangefeedDDLError=return(true)' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.72247226.out server --log-file /tmp/tidb_cdc_test/changefeed_error/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_error/cdc_data --cluster-id default + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 07:34:53 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [817 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8cc7ac14-d51b-4ac9-a24b-491f611f174b {"id":"8cc7ac14-d51b-4ac9-a24b-491f611f174b","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752090} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffb926a1dc 8cc7ac14-d51b-4ac9-a24b-491f611f174b /tidb/cdc/default/default/upstream/7489358153106219116 {"id":7489358153106219116,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8cc7ac14-d51b-4ac9-a24b-491f611f174b {"id":"8cc7ac14-d51b-4ac9-a24b-491f611f174b","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752090} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffb926a1dc 8cc7ac14-d51b-4ac9-a24b-491f611f174b /tidb/cdc/default/default/upstream/7489358153106219116 {"id":7489358153106219116,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8cc7ac14-d51b-4ac9-a24b-491f611f174b {"id":"8cc7ac14-d51b-4ac9-a24b-491f611f174b","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752090} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffb926a1dc 8cc7ac14-d51b-4ac9-a24b-491f611f174b /tidb/cdc/default/default/upstream/7489358153106219116 {"id":7489358153106219116,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x ++ date + echo '[Fri Apr 4 15:34:53 CST 2025] <<<<<< START pulsar cluster in ddl_attributes case >>>>>>' [Fri Apr 4 15:34:53 CST 2025] <<<<<< START pulsar cluster in ddl_attributes case >>>>>> + workdir=/tmp/tidb_cdc_test/ddl_attributes + cluster_type=normal + cd /tmp/tidb_cdc_test/ddl_attributes + DEFAULT_PULSAR_HOME=/usr/local/pulsar + pulsar_dir=/usr/local/pulsar ++ cat + mtls_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderTls brokerClientTlsEnabled=true brokerClientTrustCertsFilePath=/tmp/tidb_cdc_test/ddl_attributes/ca.cert.pem brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls brokerClientAuthenticationParameters={"tlsCertFile":"/tmp/tidb_cdc_test/ddl_attributes/broker_client.cert.pem","tlsKeyFile":"/tmp/tidb_cdc_test/ddl_attributes/broker_client.key-pk8.pem"} brokerServicePortTls=6651 webServicePortTls=8443 tlsTrustCertsFilePath=/tmp/tidb_cdc_test/ddl_attributes/ca.cert.pem tlsCertificateFilePath=/tmp/tidb_cdc_test/ddl_attributes/server.cert.pem tlsKeyFilePath=/tmp/tidb_cdc_test/ddl_attributes/server.key-pk8.pem tlsRequireTrustedClientCertOnConnect=true tlsAllowInsecureConnection=false tlsCertRefreshCheckDurationSec=300' ++ cat + normal_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' ++ cat + mtls_client_conf=' webServiceUrl=https://localhost:8443/ brokerServiceUrl=pulsar+ssl://localhost:6651/ authPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls authParams=tlsCertFile:/tmp/tidb_cdc_test/ddl_attributes/broker_client.cert.pem,tlsKeyFile:/tmp/tidb_cdc_test/ddl_attributes/broker_client.key-pk8.pem tlsTrustCertsFilePath=/tmp/tidb_cdc_test/ddl_attributes/ca.cert.pem' ++ cat + oauth_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/ authPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 authParams={"privateKey":"/tmp/tidb_cdc_test/ddl_attributes/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"}' ++ cat + oauth_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderToken brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 brokerClientAuthenticationParameters={"privateKey":"file:///tmp/tidb_cdc_test/ddl_attributes/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"} tokenSecretKey=data:;base64,U0poWDM2X0thcFlTeWJCdEpxMzVseFhfQnJyNExSVVJTa203UW1YSkdteThwVUZXOUVJT2NWUVBzeWt6OS1qag==' ++ cat + credential_json=' { "client_id":"1234", "client_secret":"e0KVlA2EiBfjoN13olyZd2kv1KL", "audience":"cdc-api-uri", "issuer_url":"http://localhost:9096", "type": "client_credentials" }' ++ cat + cert_server_conf='[ req ] default_bits = 2048 prompt = no default_md = sha256 distinguished_name = dn [ v3_ext ] authorityKeyIdentifier=keyid,issuer:always basicConstraints=CA:FALSE keyUsage=critical, digitalSignature, keyEncipherment extendedKeyUsage=serverAuth subjectAltName=@alt_names [ dn ] CN = server [ alt_names ] DNS.1 = localhost IP.1 = 127.0.0.1' + echo ' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' + cp /usr/local/pulsar/conf/standalone.conf /tmp/tidb_cdc_test/ddl_attributes/pulsar_standalone.conf + pulsar_port=6650 + '[' normal == mtls ']' + '[' normal == oauth ']' + echo 'no cluster type specified, using default configuration.' no cluster type specified, using default configuration. ++ date + echo '[Fri Apr 4 15:34:53 CST 2025] <<<<<< START pulsar cluster in normal mode in ddl_attributes case >>>>>>' [Fri Apr 4 15:34:53 CST 2025] <<<<<< START pulsar cluster in normal mode in ddl_attributes case >>>>>> + echo 'Waiting for pulsar port to be ready...' Waiting for pulsar port to be ready... + i=0 + nc -z localhost 6650 + /usr/local/pulsar/bin/pulsar standalone --config /tmp/tidb_cdc_test/ddl_attributes/pulsar_standalone.conf -nfw --metadata-dir /tmp/tidb_cdc_test/ddl_attributes/pulsar-metadata --bookkeeper-dir /tmp/tidb_cdc_test/ddl_attributes/pulsar-bookie + i=1 + '[' 1 -gt 20 ']' + sleep 2 table ddl_manager.finish_mark not exists for 22-th check, retry later table test.finish_mark exists check diff successfully check diff failed 1-th time, retry later table partition_table.finish_mark not exists for 2-th check, retry later + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cdc_server_tips.cli.6270.out cli tso query --pd=http://127.0.0.1:2379 check diff failed 1-th time, retry later + nc -z localhost 6650 + i=2 + '[' 2 -gt 20 ']' + sleep 2 Tenant not found Reason: Tenant not found table ddl_manager.finish_mark not exists for 23-th check, retry later + i=1 + '[' 1 -gt 20 ']' + sleep 2 start tidb cluster in /tmp/tidb_cdc_test/processor_etcd_worker_delay Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... + set +x + tso='457114149172543489 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 457114149172543489 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x table partition_table.finish_mark not exists for 3-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 07:34:56 GMT < Content-Length: 842 < Content-Type: text/plain; charset=utf-8 < { [842 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e69c0754-5fee-4ccc-afff-1e0cf2491201 {"id":"e69c0754-5fee-4ccc-afff-1e0cf2491201","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752093} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 4 /tidb/cdc/default/__cdc_meta__/owner/223195ffb896d879 e69c0754-5fee-4ccc-afff-1e0cf2491201 /tidb/cdc/default/default/upstream/7489358005347271568 {"id":7489358005347271568,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e69c0754-5fee-4ccc-afff-1e0cf2491201 {"id":"e69c0754-5fee-4ccc-afff-1e0cf2491201","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752093} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 4 /tidb/cdc/default/__cdc_meta__/owner/223195ffb896d879 e69c0754-5fee-4ccc-afff-1e0cf2491201 /tidb/cdc/default/default/upstream/7489358005347271568 {"id":7489358005347271568,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e69c0754-5fee-4ccc-afff-1e0cf2491201 {"id":"e69c0754-5fee-4ccc-afff-1e0cf2491201","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752093} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 4 /tidb/cdc/default/__cdc_meta__/owner/223195ffb896d879 e69c0754-5fee-4ccc-afff-1e0cf2491201 /tidb/cdc/default/default/upstream/7489358005347271568 {"id":7489358005347271568,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.7279.out cli changefeed create --start-ts=457114138209157121 '--sink-uri=pulsar://127.0.0.1:6650/ticdc-sink-retry-test-4567?protocol=canal-json&enable-tidb-extension=true' -c changefeed-error-1 check diff failed 2-th time, retry later Create changefeed successfully! ID: changefeed-error-1 Info: {"upstream_id":7489358005347271568,"namespace":"default","id":"changefeed-error-1","sink_uri":"pulsar://127.0.0.1:6650/ticdc-sink-retry-test-4567?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2025-04-04T15:34:56.80507228+08:00","start_ts":457114138209157121,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"pulsar_config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-10-ga306f4894","resolved_ts":457114138209157121,"checkpoint_ts":457114138209157121,"checkpoint_time":"2025-04-04 15:34:13.105"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... try a VALID cdc server command [Fri Apr 4 15:34:56 CST 2025] <<<<<< START cdc server in cdc_server_tips case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cdc_server_tips.63216323.out server --log-file /tmp/tidb_cdc_test/cdc_server_tips/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/cdc_server_tips/cdc_data --cluster-id default + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + nc -z localhost 6650 + echo 'Waiting for pulsar namespace to be ready...' Waiting for pulsar namespace to be ready... + i=0 + /usr/local/pulsar/bin/pulsar-admin namespaces list public check diff failed 2-th time, retry later table ddl_manager.finish_mark not exists for 24-th check, retry later + /usr/local/pulsar/bin/pulsar-admin namespaces list public Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + set +x check diff failed 3-th time, retry later check_changefeed_status 127.0.0.1:8300 changefeed-error-1 warning last_warning ErrExecDDLFailed + endpoint=127.0.0.1:8300 + changefeed_id=changefeed-error-1 + expected_state=warning + field=last_warning + error_pattern=ErrExecDDLFailed ++ curl 127.0.0.1:8300/api/v2/changefeeds/changefeed-error-1/status % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 252 100 252 0 0 3452 0 --:--:-- --:--:-- --:--:-- 3452 + info='{"state":"warning","resolved_ts":457114138589265926,"checkpoint_ts":457114138589265926,"last_warning":{"time":"2025-04-04T15:34:58.67798893+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrExecDDLFailed","message":"[CDC:ErrExecDDLFailed]exec DDL failed"}}' + echo '{"state":"warning","resolved_ts":457114138589265926,"checkpoint_ts":457114138589265926,"last_warning":{"time":"2025-04-04T15:34:58.67798893+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrExecDDLFailed","message":"[CDC:ErrExecDDLFailed]exec DDL failed"}}' {"state":"warning","resolved_ts":457114138589265926,"checkpoint_ts":457114138589265926,"last_warning":{"time":"2025-04-04T15:34:58.67798893+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrExecDDLFailed","message":"[CDC:ErrExecDDLFailed]exec DDL failed"}} ++ jq -r .state ++ echo '{"state":"warning","resolved_ts":457114138589265926,"checkpoint_ts":457114138589265926,"last_warning":{"time":"2025-04-04T15:34:58.67798893+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrExecDDLFailed","message":"[CDC:ErrExecDDLFailed]exec' DDL 'failed"}}' + state=warning + [[ ! warning == \w\a\r\n\i\n\g ]] + [[ -z last_warning ]] ++ jq -r .last_warning.message ++ echo '{"state":"warning","resolved_ts":457114138589265926,"checkpoint_ts":457114138589265926,"last_warning":{"time":"2025-04-04T15:34:58.67798893+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrExecDDLFailed","message":"[CDC:ErrExecDDLFailed]exec' DDL 'failed"}}' + error_msg='[CDC:ErrExecDDLFailed]exec DDL failed' + [[ ! [CDC:ErrExecDDLFailed]exec DDL failed =~ ErrExecDDLFailed ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.7335.out cli changefeed remove -c changefeed-error-1 table partition_table.finish_mark not exists for 4-th check, retry later public/default check diff failed 3-th time, retry later table ddl_manager.finish_mark not exists for 25-th check, retry later ++ date + echo '[Fri Apr 4 15:34:59 CST 2025] <<<<<< pulsar is ready >>>>>>' [Fri Apr 4 15:34:59 CST 2025] <<<<<< pulsar is ready >>>>>> + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_attributes.cli.6779.out cli changefeed create --start-ts=457114147596795905 '--sink-uri=pulsar://127.0.0.1:6650/ticdc-ddl-attributes-test-21559?protocol=canal-json&enable-tidb-extension=true' Changefeed remove successfully. ID: changefeed-error-1 CheckpointTs: 457114138589265926 SinkURI: pulsar://127.0.0.1:6650/ticdc-sink-retry-test-4567?protocol=canal-json&enable-tidb-extension=true PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 07:34:59 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [817 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/58e015c2-3412-4118-8f8f-2453ab70a835 {"id":"58e015c2-3412-4118-8f8f-2453ab70a835","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752097} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffb93a34ea 58e015c2-3412-4118-8f8f-2453ab70a835 /tidb/cdc/default/default/upstream/7489358180080460011 {"id":7489358180080460011,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/58e015c2-3412-4118-8f8f-2453ab70a835 {"id":"58e015c2-3412-4118-8f8f-2453ab70a835","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752097} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffb93a34ea 58e015c2-3412-4118-8f8f-2453ab70a835 /tidb/cdc/default/default/upstream/7489358180080460011 {"id":7489358180080460011,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/58e015c2-3412-4118-8f8f-2453ab70a835 {"id":"58e015c2-3412-4118-8f8f-2453ab70a835","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752097} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffb93a34ea 58e015c2-3412-4118-8f8f-2453ab70a835 /tidb/cdc/default/default/upstream/7489358180080460011 {"id":7489358180080460011,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x public/default Create changefeed successfully! ID: d04d63f4-726a-4f31-9d3e-d408e2da2a21 Info: {"upstream_id":7489358153106219116,"namespace":"default","id":"d04d63f4-726a-4f31-9d3e-d408e2da2a21","sink_uri":"pulsar://127.0.0.1:6650/ticdc-ddl-attributes-test-21559?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2025-04-04T15:35:00.514380858+08:00","start_ts":457114147596795905,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"pulsar_config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-10-ga306f4894","resolved_ts":457114147596795905,"checkpoint_ts":457114147596795905,"checkpoint_time":"2025-04-04 15:34:48.916"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... check diff failed 4-th time, retry later ++ date + echo '[Fri Apr 4 15:35:00 CST 2025] <<<<<< pulsar is ready >>>>>>' [Fri Apr 4 15:35:00 CST 2025] <<<<<< pulsar is ready >>>>>> table partition_table.finish_mark not exists for 5-th check, retry later + set +x check diff failed 4-th time, retry later wait process cdc.test exit for 1-th time... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489358225013489268 TiDB Cluster ID. tikv_gc_leader_uuid 657fee5f6c4000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-7fxb-ntnhx, pid:5076, start at 2025-04-04 15:35:01.315144522 +0800 CST m=+1.591884019 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:37:01.322 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:35:01.297 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:25:01.297 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Create changefeed successfully! ID: 9efbcf22-740f-4d2d-9ca7-d61de4212c5f Info: {"upstream_id":7489358145248701036,"namespace":"default","id":"9efbcf22-740f-4d2d-9ca7-d61de4212c5f","sink_uri":"pulsar+ssl://127.0.0.1:6651/ticdc-region-merge-test-23094?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2025-04-04T15:35:01.678198174+08:00","start_ts":457114150746193923,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"pulsar_config":{"tls-trust-certs-file-path":"/tmp/tidb_cdc_test/region_merge/ca.cert.pem","connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30,"auth-tls-certificate-path":"/tmp/tidb_cdc_test/region_merge/broker_client.cert.pem","auth-tls-private-key-path":"/tmp/tidb_cdc_test/region_merge/broker_client.key-pk8.pem"},"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-10-ga306f4894","resolved_ts":457114150746193923,"checkpoint_ts":457114150746193923,"checkpoint_time":"2025-04-04 15:35:00.930"} [Fri Apr 4 15:35:01 CST 2025] <<<<<< START Pulsar consumer in region_merge case >>>>>> split_and_random_merge scale: 20 *************************** 1. row *************************** count(distinct region_id): 1 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/synced_status_with_redo/run.sh using Sink-Type: pulsar... <<================= +++ dirname /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/synced_status_with_redo/run.sh ++ cd /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/synced_status_with_redo ++ pwd + CUR=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/synced_status_with_redo + source /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/synced_status_with_redo/../_utils/test_prepare ++ UP_TIDB_HOST=127.0.0.1 ++ UP_TIDB_PORT=4000 ++ UP_TIDB_OTHER_PORT=4001 ++ UP_TIDB_STATUS=10080 ++ UP_TIDB_OTHER_STATUS=10081 ++ DOWN_TIDB_HOST=127.0.0.1 ++ DOWN_TIDB_PORT=3306 ++ DOWN_TIDB_STATUS=20080 ++ TLS_TIDB_HOST=127.0.0.1 ++ TLS_TIDB_PORT=3307 ++ TLS_TIDB_STATUS=30080 ++ UP_PD_HOST_1=127.0.0.1 ++ UP_PD_PORT_1=2379 ++ UP_PD_PEER_PORT_1=2380 ++ UP_PD_HOST_2=127.0.0.1 ++ UP_PD_PORT_2=2679 ++ UP_PD_PEER_PORT_2=2680 ++ UP_PD_HOST_3=127.0.0.1 ++ UP_PD_PORT_3=2779 ++ UP_PD_PEER_PORT_3=2780 ++ DOWN_PD_HOST=127.0.0.1 ++ DOWN_PD_PORT=2479 ++ DOWN_PD_PEER_PORT=2480 ++ TLS_PD_HOST=127.0.0.1 ++ TLS_PD_PORT=2579 ++ TLS_PD_PEER_PORT=2580 ++ UP_TIKV_HOST_1=127.0.0.1 ++ UP_TIKV_PORT_1=20160 ++ UP_TIKV_STATUS_PORT_1=20181 ++ UP_TIKV_HOST_2=127.0.0.1 ++ UP_TIKV_PORT_2=20161 ++ UP_TIKV_STATUS_PORT_2=20182 ++ UP_TIKV_HOST_3=127.0.0.1 ++ UP_TIKV_PORT_3=20162 ++ UP_TIKV_STATUS_PORT_3=20183 ++ DOWN_TIKV_HOST=127.0.0.1 ++ DOWN_TIKV_PORT=21160 ++ DOWN_TIKV_STATUS_PORT=21180 ++ TLS_TIKV_HOST=127.0.0.1 ++ TLS_TIKV_PORT=22160 ++ TLS_TIKV_STATUS_PORT=22180 +++ cat /tmp/tidb_cdc_test/KAFKA_VERSION +++ echo 2.4.1 ++ KAFKA_VERSION=2.4.1 + WORK_DIR=/tmp/tidb_cdc_test/synced_status_with_redo + CDC_BINARY=cdc.test + SINK_TYPE=pulsar + CDC_COUNT=3 + DB_COUNT=4 + trap stop_tidb_cluster EXIT + run_normal_case_and_unavailable_pd conf/changefeed-redo.toml + rm -rf /tmp/tidb_cdc_test/synced_status_with_redo + mkdir -p /tmp/tidb_cdc_test/synced_status_with_redo + start_tidb_cluster --workdir /tmp/tidb_cdc_test/synced_status_with_redo The 1 times to try to start tidb cluster... + set +x [Fri Apr 4 15:35:01 CST 2025] <<<<<< START Pulsar consumer in ddl_attributes case >>>>>> VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489358225013489268 TiDB Cluster ID. tikv_gc_leader_uuid 657fee5f6c4000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-7fxb-ntnhx, pid:5076, start at 2025-04-04 15:35:01.315144522 +0800 CST m=+1.591884019 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:37:01.322 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:35:01.297 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:25:01.297 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489358228195460265 TiDB Cluster ID. tikv_gc_leader_uuid 657fee5f604000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-7fxb-ntnhx, pid:5154, start at 2025-04-04 15:35:01.292627856 +0800 CST m=+1.498185348 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:37:01.299 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:35:01.299 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:25:01.299 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/processor_etcd_worker_delay/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/processor_etcd_worker_delay/tiflash/log/error.log table ddl_manager.finish_mark not exists for 26-th check, retry later wait process cdc.test exit for 2-th time... arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/processor_etcd_worker_delay/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/processor_etcd_worker_delay/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/processor_etcd_worker_delay/tiflash/log/proxy.log"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/owner' 'owner' + key_prefix=/tidb/cdc/default/__cdc_meta__/owner + message=owner ++ etcdctl get /tidb/cdc/default/__cdc_meta__/owner --prefix --keys-only + info= + [[ '' =~ owner ]] + echo 'check pass' check pass + exit 0 run task successfully [Fri Apr 4 15:35:02 CST 2025] <<<<<< START cdc server in changefeed_error case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/tiflow/pkg/txnutil/gc/InjectActualGCSafePoint=return(9223372036854775807)' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.73827384.out server --log-file /tmp/tidb_cdc_test/changefeed_error/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_error/cdc_data --cluster-id default + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' check diff failed 5-th time, retry later + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table partition_table.finish_mark not exists for 6-th check, retry later shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory The 1 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/synced_status_with_redo Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 check diff failed 5-th time, retry later table ddl_manager.finish_mark not exists for 27-th check, retry later Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... ++ date + echo '[Fri Apr 4 15:35:04 CST 2025] <<<<<< START pulsar cluster in processor_etcd_worker_delay case >>>>>>' [Fri Apr 4 15:35:04 CST 2025] <<<<<< START pulsar cluster in processor_etcd_worker_delay case >>>>>> + workdir=/tmp/tidb_cdc_test/processor_etcd_worker_delay + cluster_type=normal + cd /tmp/tidb_cdc_test/processor_etcd_worker_delay + DEFAULT_PULSAR_HOME=/usr/local/pulsar + pulsar_dir=/usr/local/pulsar ++ cat + mtls_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderTls brokerClientTlsEnabled=true brokerClientTrustCertsFilePath=/tmp/tidb_cdc_test/processor_etcd_worker_delay/ca.cert.pem brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls brokerClientAuthenticationParameters={"tlsCertFile":"/tmp/tidb_cdc_test/processor_etcd_worker_delay/broker_client.cert.pem","tlsKeyFile":"/tmp/tidb_cdc_test/processor_etcd_worker_delay/broker_client.key-pk8.pem"} brokerServicePortTls=6651 webServicePortTls=8443 tlsTrustCertsFilePath=/tmp/tidb_cdc_test/processor_etcd_worker_delay/ca.cert.pem tlsCertificateFilePath=/tmp/tidb_cdc_test/processor_etcd_worker_delay/server.cert.pem tlsKeyFilePath=/tmp/tidb_cdc_test/processor_etcd_worker_delay/server.key-pk8.pem tlsRequireTrustedClientCertOnConnect=true tlsAllowInsecureConnection=false tlsCertRefreshCheckDurationSec=300' ++ cat + normal_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' ++ cat + mtls_client_conf=' webServiceUrl=https://localhost:8443/ brokerServiceUrl=pulsar+ssl://localhost:6651/ authPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls authParams=tlsCertFile:/tmp/tidb_cdc_test/processor_etcd_worker_delay/broker_client.cert.pem,tlsKeyFile:/tmp/tidb_cdc_test/processor_etcd_worker_delay/broker_client.key-pk8.pem tlsTrustCertsFilePath=/tmp/tidb_cdc_test/processor_etcd_worker_delay/ca.cert.pem' ++ cat + oauth_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/ authPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 authParams={"privateKey":"/tmp/tidb_cdc_test/processor_etcd_worker_delay/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"}' ++ cat + oauth_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderToken brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 brokerClientAuthenticationParameters={"privateKey":"file:///tmp/tidb_cdc_test/processor_etcd_worker_delay/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"} tokenSecretKey=data:;base64,U0poWDM2X0thcFlTeWJCdEpxMzVseFhfQnJyNExSVVJTa203UW1YSkdteThwVUZXOUVJT2NWUVBzeWt6OS1qag==' ++ cat + credential_json=' { "client_id":"1234", "client_secret":"e0KVlA2EiBfjoN13olyZd2kv1KL", "audience":"cdc-api-uri", "issuer_url":"http://localhost:9096", "type": "client_credentials" }' ++ cat + cert_server_conf='[ req ] default_bits = 2048 prompt = no default_md = sha256 distinguished_name = dn [ v3_ext ] authorityKeyIdentifier=keyid,issuer:always basicConstraints=CA:FALSE keyUsage=critical, digitalSignature, keyEncipherment extendedKeyUsage=serverAuth subjectAltName=@alt_names [ dn ] CN = server [ alt_names ] DNS.1 = localhost IP.1 = 127.0.0.1' + echo ' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' + cp /usr/local/pulsar/conf/standalone.conf /tmp/tidb_cdc_test/processor_etcd_worker_delay/pulsar_standalone.conf + pulsar_port=6650 + '[' normal == mtls ']' + '[' normal == oauth ']' + echo 'no cluster type specified, using default configuration.' no cluster type specified, using default configuration. ++ date + echo '[Fri Apr 4 15:35:04 CST 2025] <<<<<< START pulsar cluster in normal mode in processor_etcd_worker_delay case >>>>>>' [Fri Apr 4 15:35:04 CST 2025] <<<<<< START pulsar cluster in normal mode in processor_etcd_worker_delay case >>>>>> + echo 'Waiting for pulsar port to be ready...' Waiting for pulsar port to be ready... + i=0 + nc -z localhost 6650 + /usr/local/pulsar/bin/pulsar standalone --config /tmp/tidb_cdc_test/processor_etcd_worker_delay/pulsar_standalone.conf -nfw --metadata-dir /tmp/tidb_cdc_test/processor_etcd_worker_delay/pulsar-metadata --bookkeeper-dir /tmp/tidb_cdc_test/processor_etcd_worker_delay/pulsar-bookie + i=1 + '[' 1 -gt 20 ']' + sleep 2 table partition_table.finish_mark not exists for 7-th check, retry later check diff failed 6-th time, retry later check diff successfully table ddl_manager.finish_mark not exists for 28-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 07:35:05 GMT < Content-Length: 842 < Content-Type: text/plain; charset=utf-8 < { [842 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6d58887d-eb28-4543-b4bd-08d7308a7583 {"id":"6d58887d-eb28-4543-b4bd-08d7308a7583","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752102} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 5 /tidb/cdc/default/__cdc_meta__/owner/223195ffb896d8f1 6d58887d-eb28-4543-b4bd-08d7308a7583 /tidb/cdc/default/default/upstream/7489358005347271568 {"id":7489358005347271568,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6d58887d-eb28-4543-b4bd-08d7308a7583 {"id":"6d58887d-eb28-4543-b4bd-08d7308a7583","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752102} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 5 /tidb/cdc/default/__cdc_meta__/owner/223195ffb896d8f1 6d58887d-eb28-4543-b4bd-08d7308a7583 /tidb/cdc/default/default/upstream/7489358005347271568 {"id":7489358005347271568,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6d58887d-eb28-4543-b4bd-08d7308a7583 {"id":"6d58887d-eb28-4543-b4bd-08d7308a7583","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752102} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 5 /tidb/cdc/default/__cdc_meta__/owner/223195ffb896d8f1 6d58887d-eb28-4543-b4bd-08d7308a7583 /tidb/cdc/default/default/upstream/7489358005347271568 {"id":7489358005347271568,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.7437.out cli changefeed create --start-ts=457114138209157121 '--sink-uri=pulsar://127.0.0.1:6650/ticdc-sink-retry-test-4567?protocol=canal-json&enable-tidb-extension=true' -c changefeed-error-2 table ddl_attributes.attributes_t1_new not exists for 1-th check, retry later Create changefeed successfully! ID: changefeed-error-2 Info: {"upstream_id":7489358005347271568,"namespace":"default","id":"changefeed-error-2","sink_uri":"pulsar://127.0.0.1:6650/ticdc-sink-retry-test-4567?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2025-04-04T15:35:06.069611962+08:00","start_ts":457114138209157121,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"pulsar_config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-10-ga306f4894","resolved_ts":457114138209157121,"checkpoint_ts":457114138209157121,"checkpoint_time":"2025-04-04 15:34:13.105"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/batch_add_table Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... + nc -z localhost 6650 + i=2 + '[' 2 -gt 20 ']' + sleep 2 table partition_table.finish_mark exists check diff failed 1-th time, retry later check diff successfully check diff failed 7-th time, retry later wait process cdc.test exit for 1-th time... + set +x check_changefeed_state http://127.0.0.1:2379 changefeed-error-2 failed [CDC:ErrSnapshotLostByGC] + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-error-2 + expected_state=failed + error_msg='[CDC:ErrSnapshotLostByGC]' + tls_dir='[CDC:ErrSnapshotLostByGC]' + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-error-2 -s + info='{ "upstream_id": 7489358005347271568, "namespace": "default", "id": "changefeed-error-2", "state": "failed", "checkpoint_tso": 457114138209157121, "checkpoint_time": "2025-04-04 15:34:13.105", "error": { "time": "2025-04-04T15:35:06.098710666+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrSnapshotLostByGC", "message": "[CDC:ErrSnapshotLostByGC]fail to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 457114138209157121 is earlier than or equal to GC safepoint at 9223372036854775807" } }' + echo '{ "upstream_id": 7489358005347271568, "namespace": "default", "id": "changefeed-error-2", "state": "failed", "checkpoint_tso": 457114138209157121, "checkpoint_time": "2025-04-04 15:34:13.105", "error": { "time": "2025-04-04T15:35:06.098710666+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrSnapshotLostByGC", "message": "[CDC:ErrSnapshotLostByGC]fail to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 457114138209157121 is earlier than or equal to GC safepoint at 9223372036854775807" } }' { "upstream_id": 7489358005347271568, "namespace": "default", "id": "changefeed-error-2", "state": "failed", "checkpoint_tso": 457114138209157121, "checkpoint_time": "2025-04-04 15:34:13.105", "error": { "time": "2025-04-04T15:35:06.098710666+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrSnapshotLostByGC", "message": "[CDC:ErrSnapshotLostByGC]fail to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 457114138209157121 is earlier than or equal to GC safepoint at 9223372036854775807" } } ++ echo '{' '"upstream_id":' 7489358005347271568, '"namespace":' '"default",' '"id":' '"changefeed-error-2",' '"state":' '"failed",' '"checkpoint_tso":' 457114138209157121, '"checkpoint_time":' '"2025-04-04' '15:34:13.105",' '"error":' '{' '"time":' '"2025-04-04T15:35:06.098710666+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrSnapshotLostByGC",' '"message":' '"[CDC:ErrSnapshotLostByGC]fail' to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 457114138209157121 is earlier than or equal to GC safepoint at '9223372036854775807"' '}' '}' ++ jq -r .state + state=failed + [[ ! failed == \f\a\i\l\e\d ]] ++ jq -r .error.message ++ echo '{' '"upstream_id":' 7489358005347271568, '"namespace":' '"default",' '"id":' '"changefeed-error-2",' '"state":' '"failed",' '"checkpoint_tso":' 457114138209157121, '"checkpoint_time":' '"2025-04-04' '15:34:13.105",' '"error":' '{' '"time":' '"2025-04-04T15:35:06.098710666+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrSnapshotLostByGC",' '"message":' '"[CDC:ErrSnapshotLostByGC]fail' to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 457114138209157121 is earlier than or equal to GC safepoint at '9223372036854775807"' '}' '}' + message='[CDC:ErrSnapshotLostByGC]fail to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 457114138209157121 is earlier than or equal to GC safepoint at 9223372036854775807' + [[ ! [CDC:ErrSnapshotLostByGC]fail to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 457114138209157121 is earlier than or equal to GC safepoint at 9223372036854775807 =~ \[CDC:ErrSnapshotLostByGC] ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.7501.out cli changefeed remove -c changefeed-error-2 table ddl_manager.finish_mark not exists for 29-th check, retry later table ddl_attributes.attributes_t1_new not exists for 2-th check, retry later wait process cdc.test exit for 2-th time... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release wait process cdc.test exit for 3-th time... Verifying downstream PD is started... *************************** 1. row *************************** count(distinct region_id): 20 Changefeed remove successfully. ID: changefeed-error-2 CheckpointTs: 457114138209157121 SinkURI: pulsar://127.0.0.1:6650/ticdc-sink-retry-test-4567?protocol=canal-json&enable-tidb-extension=true PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... + nc -z localhost 6650 + echo 'Waiting for pulsar namespace to be ready...' Waiting for pulsar namespace to be ready... + i=0 + /usr/local/pulsar/bin/pulsar-admin namespaces list public check diff failed 8-th time, retry later cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Fri Apr 4 15:35:08 CST 2025] <<<<<< run test case partition_table success! >>>>>> check diff failed 2-th time, retry later Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... table ddl_manager.finish_mark not exists for 30-th check, retry later Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x table ddl_attributes.attributes_t1_new not exists for 3-th check, retry later Tenant not found Reason: Tenant not found wait process cdc.test exit for 1-th time... + i=1 + '[' 1 -gt 20 ']' + sleep 2 wait process cdc.test exit for 2-th time... check diff failed 3-th time, retry later check diff failed 9-th time, retry later cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 4 15:35:11 CST 2025] <<<<<< START cdc server in changefeed_error case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/redo/ChangefeedNewRedoManagerError=2*return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.75477549.out server --log-file /tmp/tidb_cdc_test/changefeed_error/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_error/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489358268633316597 TiDB Cluster ID. tikv_gc_leader_uuid 657fee6939c0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-9bld-w6hq9, pid:12063, start at 2025-04-04 15:35:11.382469875 +0800 CST m=+1.766675311 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:37:11.391 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:35:11.385 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:25:11.385 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489358268633316597 TiDB Cluster ID. tikv_gc_leader_uuid 657fee6939c0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-9bld-w6hq9, pid:12063, start at 2025-04-04 15:35:11.382469875 +0800 CST m=+1.766675311 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:37:11.391 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:35:11.385 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:25:11.385 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489358266047170175 TiDB Cluster ID. tikv_gc_leader_uuid 657fee69464000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-9bld-w6hq9, pid:12144, start at 2025-04-04 15:35:11.403157648 +0800 CST m=+1.679111518 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:37:11.413 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:35:11.385 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:25:11.385 +0800 All versions after safe point can be accessed. (DO NOT EDIT) table ddl_attributes.attributes_t1_new exists table ddl_attributes.finish_mark not exists for 1-th check, retry later Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/synced_status_with_redo/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/synced_status_with_redo/tiflash/log/error.log arg matches is ArgMatches { args: {"config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/synced_status_with_redo/tiflash-proxy.toml"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/synced_status_with_redo/tiflash/db/proxy"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/synced_status_with_redo/tiflash/log/proxy.log"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table ddl_manager.finish_mark not exists for 31-th check, retry later + /usr/local/pulsar/bin/pulsar-admin namespaces list public check diff failed 4-th time, retry later check diff successfully wait process cdc.test exit for 1-th time... split_and_random_merge scale: 40 *************************** 1. row *************************** count(distinct region_id): 8 table ddl_attributes.finish_mark not exists for 2-th check, retry later wait process cdc.test exit for 2-th time... table ddl_manager.finish_mark not exists for 32-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489358268149927799 TiDB Cluster ID. tikv_gc_leader_uuid 657fee6ae4c0008 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-j3lw-c6tm2, pid:4648, start at 2025-04-04 15:35:13.055118731 +0800 CST m=+1.407980395 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:37:13.062 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:35:13.043 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:25:13.043 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489358268149927799 TiDB Cluster ID. tikv_gc_leader_uuid 657fee6ae4c0008 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-j3lw-c6tm2, pid:4648, start at 2025-04-04 15:35:13.055118731 +0800 CST m=+1.407980395 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:37:13.062 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:35:13.043 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:25:13.043 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489358279286639456 TiDB Cluster ID. tikv_gc_leader_uuid 657fee6b0900007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-j3lw-c6tm2, pid:4728, start at 2025-04-04 15:35:13.200086681 +0800 CST m=+1.463656614 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:37:13.207 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:35:13.188 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:25:13.188 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/batch_add_table/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/batch_add_table/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/batch_add_table/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/batch_add_table/tiflash-proxy.toml"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/batch_add_table/tiflash/db/proxy"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + cd /tmp/tidb_cdc_test/synced_status_with_redo ++ run_cdc_cli_tso_query 127.0.0.1 2379 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status_with_redo.cli.13471.out cli tso query --pd=http://127.0.0.1:2379 wait process cdc.test exit for 3-th time... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 07:35:14 GMT < Content-Length: 842 < Content-Type: text/plain; charset=utf-8 < { [842 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/fd2c7234-0442-4625-8ef4-bd505cb0d1fd {"id":"fd2c7234-0442-4625-8ef4-bd505cb0d1fd","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752111} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 7 /tidb/cdc/default/__cdc_meta__/owner/223195ffb896d970 fd2c7234-0442-4625-8ef4-bd505cb0d1fd /tidb/cdc/default/default/upstream/7489358005347271568 {"id":7489358005347271568,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/fd2c7234-0442-4625-8ef4-bd505cb0d1fd {"id":"fd2c7234-0442-4625-8ef4-bd505cb0d1fd","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752111} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 7 /tidb/cdc/default/__cdc_meta__/owner/223195ffb896d970 fd2c7234-0442-4625-8ef4-bd505cb0d1fd /tidb/cdc/default/default/upstream/7489358005347271568 {"id":7489358005347271568,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/fd2c7234-0442-4625-8ef4-bd505cb0d1fd {"id":"fd2c7234-0442-4625-8ef4-bd505cb0d1fd","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752111} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 7 /tidb/cdc/default/__cdc_meta__/owner/223195ffb896d970 fd2c7234-0442-4625-8ef4-bd505cb0d1fd /tidb/cdc/default/default/upstream/7489358005347271568 {"id":7489358005347271568,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.7600.out cli changefeed create --start-ts=0 '--sink-uri=pulsar://127.0.0.1:6650/ticdc-sink-retry-test-4567?protocol=canal-json&enable-tidb-extension=true' -c changefeed-initialize-error Create changefeed successfully! ID: changefeed-initialize-error Info: {"upstream_id":7489358005347271568,"namespace":"default","id":"changefeed-initialize-error","sink_uri":"pulsar://127.0.0.1:6650/ticdc-sink-retry-test-4567?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2025-04-04T15:35:14.877252604+08:00","start_ts":457114154370596869,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"pulsar_config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-10-ga306f4894","resolved_ts":457114154370596869,"checkpoint_ts":457114154370596869,"checkpoint_time":"2025-04-04 15:35:14.756"} PASS public/default ++ date + echo '[Fri Apr 4 15:35:14 CST 2025] <<<<<< pulsar is ready >>>>>>' [Fri Apr 4 15:35:14 CST 2025] <<<<<< pulsar is ready >>>>>> [Fri Apr 4 15:35:14 CST 2025] <<<<<< START cdc server in processor_etcd_worker_delay case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/pkg/orchestrator/ProcessorEtcdDelay=return(true)' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.processor_etcd_worker_delay.69756977.out server --log-file /tmp/tidb_cdc_test/processor_etcd_worker_delay/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/processor_etcd_worker_delay/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 coverage: 2.5% of statements in github.com/pingcap/tiflow/... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Fri Apr 4 15:35:15 CST 2025] <<<<<< run test case canal_json_content_compatible success! >>>>>> check diff failed 5-th time, retry later + set +x + tso='457114154286448641 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 457114154286448641 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + start_ts=457114154286448641 + run_cdc_server --workdir /tmp/tidb_cdc_test/synced_status_with_redo --binary cdc.test [Fri Apr 4 15:35:15 CST 2025] <<<<<< START cdc server in synced_status_with_redo case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status_with_redo.1350013502.out server --log-file /tmp/tidb_cdc_test/synced_status_with_redo/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/synced_status_with_redo/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table ddl_manager.finish_mark not exists for 33-th check, retry later [Fri Apr 4 15:35:16 CST 2025] <<<<<< START cdc server in batch_add_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.batch_add_table.61956197.out server --log-file /tmp/tidb_cdc_test/batch_add_table/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/batch_add_table/cdc_data --cluster-id default + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table ddl_attributes.finish_mark not exists for 3-th check, retry later + set +x check_changefeed_state http://127.0.0.1:2379 changefeed-initialize-error normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-initialize-error + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-initialize-error -s + info='{ "upstream_id": 7489358005347271568, "namespace": "default", "id": "changefeed-initialize-error", "state": "warning", "checkpoint_tso": 457114154370596869, "checkpoint_time": "2025-04-04 15:35:14.756", "error": { "time": "2025-04-04T15:35:15.054383989+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } }' + echo '{ "upstream_id": 7489358005347271568, "namespace": "default", "id": "changefeed-initialize-error", "state": "warning", "checkpoint_tso": 457114154370596869, "checkpoint_time": "2025-04-04 15:35:14.756", "error": { "time": "2025-04-04T15:35:15.054383989+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } }' { "upstream_id": 7489358005347271568, "namespace": "default", "id": "changefeed-initialize-error", "state": "warning", "checkpoint_tso": 457114154370596869, "checkpoint_time": "2025-04-04 15:35:14.756", "error": { "time": "2025-04-04T15:35:15.054383989+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } } ++ echo '{' '"upstream_id":' 7489358005347271568, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"warning",' '"checkpoint_tso":' 457114154370596869, '"checkpoint_time":' '"2025-04-04' '15:35:14.756",' '"error":' '{' '"time":' '"2025-04-04T15:35:15.054383989+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrProcessorUnknown",' '"message":' '"changefeed' new redo manager injected 'error"' '}' '}' ++ jq -r .state + state=warning + [[ ! warning == \n\o\r\m\a\l ]] + echo 'changefeed state warning does not equal to normal' changefeed state warning does not equal to normal + exit 1 run task failed 1-th time, retry later check diff successfully table ddl_manager.finish_mark not exists for 34-th check, retry later table ddl_attributes.finish_mark not exists for 4-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 07:35:18 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [817 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/32e68f41-5708-4f23-af02-9b79f4f4351d {"id":"32e68f41-5708-4f23-af02-9b79f4f4351d","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752115} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffb9695b06 32e68f41-5708-4f23-af02-9b79f4f4351d /tidb/cdc/default/default/upstream/7489358225013489268 {"id":7489358225013489268,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/32e68f41-5708-4f23-af02-9b79f4f4351d {"id":"32e68f41-5708-4f23-af02-9b79f4f4351d","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752115} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffb9695b06 32e68f41-5708-4f23-af02-9b79f4f4351d /tidb/cdc/default/default/upstream/7489358225013489268 {"id":7489358225013489268,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/32e68f41-5708-4f23-af02-9b79f4f4351d {"id":"32e68f41-5708-4f23-af02-9b79f4f4351d","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752115} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffb9695b06 32e68f41-5708-4f23-af02-9b79f4f4351d /tidb/cdc/default/default/upstream/7489358225013489268 {"id":7489358225013489268,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x check diff failed 1-th time, retry later check_changefeed_state http://127.0.0.1:2379 changefeed-initialize-error normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-initialize-error + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-initialize-error -s + info='{ "upstream_id": 7489358005347271568, "namespace": "default", "id": "changefeed-initialize-error", "state": "warning", "checkpoint_tso": 457114154370596869, "checkpoint_time": "2025-04-04 15:35:14.756", "error": { "time": "2025-04-04T15:35:15.054383989+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } }' + echo '{ "upstream_id": 7489358005347271568, "namespace": "default", "id": "changefeed-initialize-error", "state": "warning", "checkpoint_tso": 457114154370596869, "checkpoint_time": "2025-04-04 15:35:14.756", "error": { "time": "2025-04-04T15:35:15.054383989+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } }' { "upstream_id": 7489358005347271568, "namespace": "default", "id": "changefeed-initialize-error", "state": "warning", "checkpoint_tso": 457114154370596869, "checkpoint_time": "2025-04-04 15:35:14.756", "error": { "time": "2025-04-04T15:35:15.054383989+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } } ++ echo '{' '"upstream_id":' 7489358005347271568, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"warning",' '"checkpoint_tso":' 457114154370596869, '"checkpoint_time":' '"2025-04-04' '15:35:14.756",' '"error":' '{' '"time":' '"2025-04-04T15:35:15.054383989+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrProcessorUnknown",' '"message":' '"changefeed' new redo manager injected 'error"' '}' '}' ++ jq -r .state + state=warning + [[ ! warning == \n\o\r\m\a\l ]] + echo 'changefeed state warning does not equal to normal' changefeed state warning does not equal to normal + exit 1 run task failed 2-th time, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/ddl_with_exists/run.sh using Sink-Type: pulsar... <<================= [Fri Apr 4 15:35:18 CST 2025] <<<<<< run test case ddl_with_exists success! >>>>>> [Fri Apr 4 15:35:18 CST 2025] <<<<<< START Pulsar consumer in processor_etcd_worker_delay case >>>>>> + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 07:35:18 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [817 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/74aa5d56-2a93-4541-bff3-6f59eacde300 {"id":"74aa5d56-2a93-4541-bff3-6f59eacde300","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752116} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffb98829d5 74aa5d56-2a93-4541-bff3-6f59eacde300 /tidb/cdc/default/default/upstream/7489358268633316597 {"id":7489358268633316597,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/74aa5d56-2a93-4541-bff3-6f59eacde300 {"id":"74aa5d56-2a93-4541-bff3-6f59eacde300","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752116} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffb98829d5 74aa5d56-2a93-4541-bff3-6f59eacde300 /tidb/cdc/default/default/upstream/7489358268633316597 {"id":7489358268633316597,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/74aa5d56-2a93-4541-bff3-6f59eacde300 {"id":"74aa5d56-2a93-4541-bff3-6f59eacde300","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752116} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffb98829d5 74aa5d56-2a93-4541-bff3-6f59eacde300 /tidb/cdc/default/default/upstream/7489358268633316597 {"id":7489358268633316597,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + config_path=conf/changefeed-redo.toml + SINK_URI='mysql://root@127.0.0.1:3306/?max-txn-row=1' + run_cdc_cli changefeed create --start-ts=457114154286448641 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/synced_status_with_redo/conf/changefeed-redo.toml + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status_with_redo.cli.13555.out cli changefeed create --start-ts=457114154286448641 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/synced_status_with_redo/conf/changefeed-redo.toml + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 07:35:19 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [817 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b9ebbced-a20e-4281-8a70-663d42fc4033 {"id":"b9ebbced-a20e-4281-8a70-663d42fc4033","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752116} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffb991f6d7 b9ebbced-a20e-4281-8a70-663d42fc4033 /tidb/cdc/default/default/upstream/7489358268149927799 {"id":7489358268149927799,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b9ebbced-a20e-4281-8a70-663d42fc4033 {"id":"b9ebbced-a20e-4281-8a70-663d42fc4033","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752116} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffb991f6d7 b9ebbced-a20e-4281-8a70-663d42fc4033 /tidb/cdc/default/default/upstream/7489358268149927799 {"id":7489358268149927799,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b9ebbced-a20e-4281-8a70-663d42fc4033 {"id":"b9ebbced-a20e-4281-8a70-663d42fc4033","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752116} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffb991f6d7 b9ebbced-a20e-4281-8a70-663d42fc4033 /tidb/cdc/default/default/upstream/7489358268149927799 {"id":7489358268149927799,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x ++ date + echo '[Fri Apr 4 15:35:19 CST 2025] <<<<<< START pulsar cluster in batch_add_table case >>>>>>' [Fri Apr 4 15:35:19 CST 2025] <<<<<< START pulsar cluster in batch_add_table case >>>>>> + workdir=/tmp/tidb_cdc_test/batch_add_table + cluster_type=normal + cd /tmp/tidb_cdc_test/batch_add_table + DEFAULT_PULSAR_HOME=/usr/local/pulsar + pulsar_dir=/usr/local/pulsar ++ cat + mtls_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderTls brokerClientTlsEnabled=true brokerClientTrustCertsFilePath=/tmp/tidb_cdc_test/batch_add_table/ca.cert.pem brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls brokerClientAuthenticationParameters={"tlsCertFile":"/tmp/tidb_cdc_test/batch_add_table/broker_client.cert.pem","tlsKeyFile":"/tmp/tidb_cdc_test/batch_add_table/broker_client.key-pk8.pem"} brokerServicePortTls=6651 webServicePortTls=8443 tlsTrustCertsFilePath=/tmp/tidb_cdc_test/batch_add_table/ca.cert.pem tlsCertificateFilePath=/tmp/tidb_cdc_test/batch_add_table/server.cert.pem tlsKeyFilePath=/tmp/tidb_cdc_test/batch_add_table/server.key-pk8.pem tlsRequireTrustedClientCertOnConnect=true tlsAllowInsecureConnection=false tlsCertRefreshCheckDurationSec=300' ++ cat + normal_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' ++ cat + mtls_client_conf=' webServiceUrl=https://localhost:8443/ brokerServiceUrl=pulsar+ssl://localhost:6651/ authPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls authParams=tlsCertFile:/tmp/tidb_cdc_test/batch_add_table/broker_client.cert.pem,tlsKeyFile:/tmp/tidb_cdc_test/batch_add_table/broker_client.key-pk8.pem tlsTrustCertsFilePath=/tmp/tidb_cdc_test/batch_add_table/ca.cert.pem' ++ cat + oauth_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/ authPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 authParams={"privateKey":"/tmp/tidb_cdc_test/batch_add_table/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"}' ++ cat + oauth_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderToken brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 brokerClientAuthenticationParameters={"privateKey":"file:///tmp/tidb_cdc_test/batch_add_table/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"} tokenSecretKey=data:;base64,U0poWDM2X0thcFlTeWJCdEpxMzVseFhfQnJyNExSVVJTa203UW1YSkdteThwVUZXOUVJT2NWUVBzeWt6OS1qag==' ++ cat + credential_json=' { "client_id":"1234", "client_secret":"e0KVlA2EiBfjoN13olyZd2kv1KL", "audience":"cdc-api-uri", "issuer_url":"http://localhost:9096", "type": "client_credentials" }' ++ cat + cert_server_conf='[ req ] default_bits = 2048 prompt = no default_md = sha256 distinguished_name = dn [ v3_ext ] authorityKeyIdentifier=keyid,issuer:always basicConstraints=CA:FALSE keyUsage=critical, digitalSignature, keyEncipherment extendedKeyUsage=serverAuth subjectAltName=@alt_names [ dn ] CN = server [ alt_names ] DNS.1 = localhost IP.1 = 127.0.0.1' + echo ' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' + cp /usr/local/pulsar/conf/standalone.conf /tmp/tidb_cdc_test/batch_add_table/pulsar_standalone.conf + pulsar_port=6650 + '[' normal == mtls ']' + '[' normal == oauth ']' + echo 'no cluster type specified, using default configuration.' no cluster type specified, using default configuration. ++ date + echo '[Fri Apr 4 15:35:19 CST 2025] <<<<<< START pulsar cluster in normal mode in batch_add_table case >>>>>>' [Fri Apr 4 15:35:19 CST 2025] <<<<<< START pulsar cluster in normal mode in batch_add_table case >>>>>> + echo 'Waiting for pulsar port to be ready...' Waiting for pulsar port to be ready... + i=0 + /usr/local/pulsar/bin/pulsar standalone --config /tmp/tidb_cdc_test/batch_add_table/pulsar_standalone.conf -nfw --metadata-dir /tmp/tidb_cdc_test/batch_add_table/pulsar-metadata --bookkeeper-dir /tmp/tidb_cdc_test/batch_add_table/pulsar-bookie + nc -z localhost 6650 + i=1 + '[' 1 -gt 20 ']' + sleep 2 Create changefeed successfully! ID: test-1 Info: {"upstream_id":7489358268633316597,"namespace":"default","id":"test-1","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2025-04-04T15:35:19.319505018+08:00","start_ts":457114154286448641,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"eventual","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"storage":"file:///tmp/tidb_cdc_test/synced_status/redo","use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":120,"checkpoint_interval":20}},"state":"normal","creator_version":"v9.0.0-beta.1-10-ga306f4894","resolved_ts":457114154286448641,"checkpoint_ts":457114154286448641,"checkpoint_time":"2025-04-04 15:35:14.435"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... table ddl_manager.finish_mark not exists for 35-th check, retry later table ddl_attributes.finish_mark not exists for 5-th check, retry later *************************** 1. row *************************** count(distinct region_id): 40 + set +x ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 221 100 221 0 0 44200 0 --:--:-- --:--:-- --:--:-- 44200 + synced_status='{"synced":true,"sink_checkpoint_ts":"2025-04-04 15:35:14.435","puller_resolved_ts":"1970-01-01 08:00:00.000","last_synced_ts":"1970-01-01 08:00:00.000","now_ts":"2025-04-04 15:35:20.000","info":"Data syncing is finished"}' ++ echo '{"synced":true,"sink_checkpoint_ts":"2025-04-04' '15:35:14.435","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2025-04-04' '15:35:20.000","info":"Data' syncing is 'finished"}' ++ jq .synced + status=true ++ echo '{"synced":true,"sink_checkpoint_ts":"2025-04-04' '15:35:14.435","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2025-04-04' '15:35:20.000","info":"Data' syncing is 'finished"}' ++ jq -r .sink_checkpoint_ts + sink_checkpoint_ts='2025-04-04 15:35:14.435' ++ echo '{"synced":true,"sink_checkpoint_ts":"2025-04-04' '15:35:14.435","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2025-04-04' '15:35:20.000","info":"Data' syncing is 'finished"}' ++ jq -r .puller_resolved_ts + puller_resolved_ts='1970-01-01 08:00:00.000' ++ echo '{"synced":true,"sink_checkpoint_ts":"2025-04-04' '15:35:14.435","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2025-04-04' '15:35:20.000","info":"Data' syncing is 'finished"}' ++ jq -r .last_synced_ts + last_synced_ts='1970-01-01 08:00:00.000' + '[' true '!=' true ']' + '[' '1970-01-01 08:00:00.000' '!=' '1970-01-01 08:00:00.000' ']' + '[' '1970-01-01 08:00:00.000' '!=' '1970-01-01 08:00:00.000' ']' ++ date '+%Y-%m-%d %H:%M:%S' + current='2025-04-04 15:35:20' + echo 'sink_checkpoint_ts is 2025-04-04' 15:35:14.435 sink_checkpoint_ts is 2025-04-04 15:35:14.435 ++ date -d '2025-04-04 15:35:14.435' +%s + checkpoint_timestamp=1743752114 ++ date -d '2025-04-04 15:35:20' +%s + current_timestamp=1743752120 + '[' 6 -gt 300 ']' + run_sql 'USE TEST;Create table t1(a int primary key, b int);insert into t1 values(1,2);insert into t1 values(2,3);' check diff failed 2-th time, retry later + check_table_exists test.t1 127.0.0.1 3306 table test.t1 not exists for 1-th check, retry later + nc -z localhost 6650 + i=2 + '[' 2 -gt 20 ']' + sleep 2 table ddl_attributes.finish_mark not exists for 6-th check, retry later valid ~~~ running cdc Failed to start cdc, the usage tips should be printed 1st test case cdc_server_tips success! try an INVALID cdc server command [Fri Apr 4 15:35:19 CST 2025] <<<<<< START cdc server in cdc_server_tips case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cdc_server_tips.64366438.out server --log-file /tmp/tidb_cdc_test/cdc_server_tips/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/cdc_server_tips/cdc_data --cluster-id default --pd None + [[ true != \n\o ]] + set +x \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-3484/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } table ddl_manager.finish_mark not exists for 36-th check, retry later [Pipeline] // dir [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } check diff failed 3-th time, retry later table test.t1 exists + sleep 5 [Pipeline] // podTemplate [Pipeline] } check_changefeed_state http://127.0.0.1:2379 changefeed-initialize-error normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-initialize-error + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-initialize-error -s + info='{ "upstream_id": 7489358005347271568, "namespace": "default", "id": "changefeed-initialize-error", "state": "normal", "checkpoint_tso": 457114156388843526, "checkpoint_time": "2025-04-04 15:35:22.455", "error": null }' + echo '{ "upstream_id": 7489358005347271568, "namespace": "default", "id": "changefeed-initialize-error", "state": "normal", "checkpoint_tso": 457114156388843526, "checkpoint_time": "2025-04-04 15:35:22.455", "error": null }' { "upstream_id": 7489358005347271568, "namespace": "default", "id": "changefeed-initialize-error", "state": "normal", "checkpoint_tso": 457114156388843526, "checkpoint_time": "2025-04-04 15:35:22.455", "error": null } ++ echo '{' '"upstream_id":' 7489358005347271568, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"normal",' '"checkpoint_tso":' 457114156388843526, '"checkpoint_time":' '"2025-04-04' '15:35:22.455",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7489358005347271568, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"normal",' '"checkpoint_tso":' 457114156388843526, '"checkpoint_time":' '"2025-04-04' '15:35:22.455",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.7750.out cli changefeed pause -c changefeed-initialize-error [Pipeline] // withEnv [Pipeline] } table processor_delay.t1 exists table processor_delay.t2 not exists for 1-th check, retry later [Pipeline] // stage [Pipeline] } PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... + nc -z localhost 6650 + echo 'Waiting for pulsar namespace to be ready...' Waiting for pulsar namespace to be ready... + i=0 + /usr/local/pulsar/bin/pulsar-admin namespaces list public table ddl_attributes.finish_mark not exists for 7-th check, retry later table ddl_manager.finish_mark not exists for 37-th check, retry later check diff failed 4-th time, retry later + set +x check_changefeed_state http://127.0.0.1:2379 changefeed-initialize-error stopped null + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-initialize-error + expected_state=stopped + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-initialize-error -s table processor_delay.t2 not exists for 2-th check, retry later + info='{ "upstream_id": 7489358005347271568, "namespace": "default", "id": "changefeed-initialize-error", "state": "stopped", "checkpoint_tso": 457114156388843526, "checkpoint_time": "2025-04-04 15:35:22.455", "error": null }' + echo '{ "upstream_id": 7489358005347271568, "namespace": "default", "id": "changefeed-initialize-error", "state": "stopped", "checkpoint_tso": 457114156388843526, "checkpoint_time": "2025-04-04 15:35:22.455", "error": null }' { "upstream_id": 7489358005347271568, "namespace": "default", "id": "changefeed-initialize-error", "state": "stopped", "checkpoint_tso": 457114156388843526, "checkpoint_time": "2025-04-04 15:35:22.455", "error": null } ++ echo '{' '"upstream_id":' 7489358005347271568, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"stopped",' '"checkpoint_tso":' 457114156388843526, '"checkpoint_time":' '"2025-04-04' '15:35:22.455",' '"error":' null '}' ++ jq -r .state + state=stopped + [[ ! stopped == \s\t\o\p\p\e\d ]] ++ echo '{' '"upstream_id":' 7489358005347271568, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"stopped",' '"checkpoint_tso":' 457114156388843526, '"checkpoint_time":' '"2025-04-04' '15:35:22.455",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.7814.out cli changefeed resume -c changefeed-initialize-error =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/multi_topics/run.sh using Sink-Type: pulsar... <<================= [Fri Apr 4 15:35:25 CST 2025] <<<<<< run test case multi_topics success! >>>>>> split_and_random_merge scale: 80 *************************** 1. row *************************** count(distinct region_id): 4 public/default ++ date + echo '[Fri Apr 4 15:35:25 CST 2025] <<<<<< pulsar is ready >>>>>>' [Fri Apr 4 15:35:25 CST 2025] <<<<<< pulsar is ready >>>>>> + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.batch_add_table.cli.6655.out cli changefeed create '--sink-uri=pulsar://127.0.0.1:6650/ticdc-batch-add-table-test-32037?protocol=canal-json&enable-tidb-extension=true' --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/batch_add_table/conf/changefeed.toml PASS coverage: 2.1% of statements in github.com/pingcap/tiflow/... table ddl_manager.finish_mark not exists for 38-th check, retry later table ddl_attributes.finish_mark not exists for 8-th check, retry later Create changefeed successfully! ID: f1971c45-c039-4ca7-a205-42ca1cc5e802 Info: {"upstream_id":7489358268149927799,"namespace":"default","id":"f1971c45-c039-4ca7-a205-42ca1cc5e802","sink_uri":"pulsar://127.0.0.1:6650/ticdc-batch-add-table-test-32037?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2025-04-04T15:35:26.556809218+08:00","start_ts":457114157303201798,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*","!test.t_1"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"pulsar_config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-10-ga306f4894","resolved_ts":457114157303201798,"checkpoint_ts":457114157303201798,"checkpoint_time":"2025-04-04 15:35:25.943"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... table processor_delay.t2 not exists for 3-th check, retry later + set +x check_changefeed_state http://127.0.0.1:2379 changefeed-initialize-error normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-initialize-error + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-initialize-error -s check diff failed 5-th time, retry later + info='{ "upstream_id": 7489358005347271568, "namespace": "default", "id": "changefeed-initialize-error", "state": "normal", "checkpoint_tso": 457114156388843526, "checkpoint_time": "2025-04-04 15:35:22.455", "error": null }' + echo '{ "upstream_id": 7489358005347271568, "namespace": "default", "id": "changefeed-initialize-error", "state": "normal", "checkpoint_tso": 457114156388843526, "checkpoint_time": "2025-04-04 15:35:22.455", "error": null }' { "upstream_id": 7489358005347271568, "namespace": "default", "id": "changefeed-initialize-error", "state": "normal", "checkpoint_tso": 457114156388843526, "checkpoint_time": "2025-04-04 15:35:22.455", "error": null } ++ echo '{' '"upstream_id":' 7489358005347271568, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"normal",' '"checkpoint_tso":' 457114156388843526, '"checkpoint_time":' '"2025-04-04' '15:35:22.455",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7489358005347271568, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"normal",' '"checkpoint_tso":' 457114156388843526, '"checkpoint_time":' '"2025-04-04' '15:35:22.455",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.7877.out cli changefeed remove -c changefeed-initialize-error table ddl_manager.finish_mark not exists for 39-th check, retry later ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 243 100 243 0 0 2612 0 --:--:-- --:--:-- --:--:-- 2612 + synced_status='{"synced":false,"sink_checkpoint_ts":"2025-04-04 15:35:26.635","puller_resolved_ts":"2025-04-04 15:35:20.785","last_synced_ts":"2025-04-04 15:35:20.785","now_ts":"2025-04-04 15:35:27.000","info":"The data syncing is not finished, please wait"}' ++ echo '{"synced":false,"sink_checkpoint_ts":"2025-04-04' '15:35:26.635","puller_resolved_ts":"2025-04-04' '15:35:20.785","last_synced_ts":"2025-04-04' '15:35:20.785","now_ts":"2025-04-04' '15:35:27.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq .synced + status=false + '[' false '!=' false ']' ++ echo '{"synced":false,"sink_checkpoint_ts":"2025-04-04' '15:35:26.635","puller_resolved_ts":"2025-04-04' '15:35:20.785","last_synced_ts":"2025-04-04' '15:35:20.785","now_ts":"2025-04-04' '15:35:27.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq -r .info + info='The data syncing is not finished, please wait' + '[' 'The data syncing is not finished, please wait' '!=' 'The data syncing is not finished, please wait' ']' + sleep 130 + set +x [Fri Apr 4 15:35:27 CST 2025] <<<<<< START Pulsar consumer in batch_add_table case >>>>>> table test.t_1 exists table test.t_2 not exists for 1-th check, retry later table ddl_attributes.finish_mark not exists for 9-th check, retry later Changefeed remove successfully. ID: changefeed-initialize-error CheckpointTs: 457114156388843526 SinkURI: pulsar://127.0.0.1:6650/ticdc-sink-retry-test-4567?protocol=canal-json&enable-tidb-extension=true PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/avro_basic/run.sh using Sink-Type: pulsar... <<================= [Fri Apr 4 15:35:28 CST 2025] <<<<<< run test case avro_basic success! >>>>>> table processor_delay.t2 exists table processor_delay.t3 not exists for 1-th check, retry later check diff failed 6-th time, retry later + set +x wait process cdc.test exit for 1-th time... table ddl_manager.finish_mark not exists for 40-th check, retry later table ddl_attributes.finish_mark exists check diff successfully wait process cdc.test exit for 2-th time... table processor_delay.t3 exists table processor_delay.t4 not exists for 1-th check, retry later table test.t_2 exists table test.t_3 exists table test.t_4 exists table test.t_5 exists table test.t_6 exists table test.t_7 exists table test.t_8 exists table test.t_9 exists table test.t_10 exists table test.t_11 exists table test.t_12 exists table test.t_13 exists table test.t_14 exists table test.t_15 exists table test.t_16 exists table test.t_17 exists table test.t_18 exists table test.t_19 exists table test.t_20 exists table test.t_21 exists table test.t_22 exists table test.t_23 exists table test.t_24 exists table test.t_25 exists table test.t_26 exists table test.t_27 exists table test.t_28 exists table test.t_29 exists table test.t_30 exists table test.t_31 exists table test.t_32 exists table test.t_33 exists table test.t_34 exists table test.t_35 exists table test.t_36 exists table test.t_37 exists table test.t_38 exists table test.t_39 exists table test.t_40 exists table test.t_41 exists table test.t_42 exists table test.t_43 exists table test.t_44 exists table test.t_45 exists table test.t_46 exists table test.t_47 exists table test.t_48 exists table test.t_49 exists table test.t_50 exists table test.t_51 exists table test.t_52 exists table test.t_53 exists table test.t_54 exists table test.t_55 exists table test.t_56 exists table test.t_57 exists table test.t_58 exists table test.t_59 exists table test.t_60 exists table test.t_61 exists table test.t_62 exists table test.t_63 exists table test.t_64 exists table test.t_65 exists wait process cdc.test exit for 1-th time... *************************** 1. row *************************** count(distinct region_id): 78 table test.t_66 exists table test.t_67 exists table test.t_68 exists table test.t_69 exists table test.t_70 exists table test.t_71 exists table test.t_72 exists table test.t_73 exists table test.t_74 exists table test.t_75 exists table test.t_76 exists table test.t_77 exists table test.t_78 exists table test.t_79 exists table test.t_80 exists table test.t_81 exists table test.t_82 exists table test.t_83 exists table test.t_84 exists table test.t_85 exists table test.t_86 exists table test.t_87 exists table test.t_88 exists table test.t_89 exists table test.t_90 exists table test.t_91 exists table test.t_92 exists table test.t_93 exists table test.t_94 exists table test.t_95 exists table test.t_96 exists table test.t_97 exists cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 4 15:35:30 CST 2025] <<<<<< run test case changefeed_error success! >>>>>> check diff successfully table test.t_98 exists table test.t_99 exists table test.t_100 exists wait process cdc.test exit for 2-th time... table test.t_100 not exists for 1-th check, retry later cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 4 15:35:31 CST 2025] <<<<<< run test case ddl_attributes success! >>>>>> table ddl_manager.finish_mark not exists for 41-th check, retry later check diff failed 1-th time, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/canal_json_handle_key_only/run.sh using Sink-Type: pulsar... <<================= [Fri Apr 4 15:35:31 CST 2025] <<<<<< run test case canal_json_handle_key_only success! >>>>>> table processor_delay.t4 exists table processor_delay.t5 not exists for 1-th check, retry later table test.t_100 not exists for 2-th check, retry later check diff failed 2-th time, retry later table ddl_manager.finish_mark not exists for 42-th check, retry later table processor_delay.t5 exists table processor_delay.t6 not exists for 1-th check, retry later table region_merge.t1 exists check diff failed 1-th time, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/open_protocol_handle_key_only/run.sh using Sink-Type: pulsar... <<================= [Fri Apr 4 15:35:35 CST 2025] <<<<<< run test case open_protocol_handle_key_only success! >>>>>> table test.t_100 not exists for 3-th check, retry later table ddl_manager.finish_mark not exists for 43-th check, retry later check diff failed 3-th time, retry later table processor_delay.t6 exists table processor_delay.t7 not exists for 1-th check, retry later table test.t_100 not exists for 4-th check, retry later table ddl_manager.finish_mark not exists for 44-th check, retry later check diff failed 2-th time, retry later check diff failed 4-th time, retry later table processor_delay.t7 exists table processor_delay.t8 not exists for 1-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/canal_json_claim_check/run.sh using Sink-Type: pulsar... <<================= [Fri Apr 4 15:35:38 CST 2025] <<<<<< run test case canal_json_claim_check success! >>>>>> table test.t_100 not exists for 5-th check, retry later table ddl_manager.finish_mark not exists for 45-th check, retry later invalid ~~~ running cdc Failed to start cdc, the usage tips should be printed 2nd test case cdc_server_tips success! [Fri Apr 4 15:35:40 CST 2025] <<<<<< run all test cases cdc_server_tips success! >>>>>> check diff failed 3-th time, retry later check diff failed 5-th time, retry later table processor_delay.t8 exists table processor_delay.t9 not exists for 1-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/ddl_sequence/run.sh using Sink-Type: pulsar... <<================= The 1 times to try to start tidb cluster... table test.t_100 not exists for 6-th check, retry later \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-3484/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } table ddl_manager.finish_mark not exists for 46-th check, retry later [Pipeline] // dir [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/open_protocol_claim_check/run.sh using Sink-Type: pulsar... <<================= [Fri Apr 4 15:35:42 CST 2025] <<<<<< run test case open_protocol_claim_check success! >>>>>> [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } check diff failed 4-th time, retry later check diff successfully table processor_delay.t9 exists table processor_delay.t10 not exists for 1-th check, retry later wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... table test.t_100 exists table batch_add_table.finish_mark not exists for 1-th check, retry later table ddl_manager.finish_mark not exists for 47-th check, retry later cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 4 15:35:44 CST 2025] <<<<<< run test case changefeed_pause_resume success! >>>>>> check diff successfully wait process cdc.test exit for 1-th time... table processor_delay.t10 exists table processor_delay.t11 not exists for 1-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/ddl_sequence Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 4 15:35:45 CST 2025] <<<<<< run test case region_merge success! >>>>>> table ddl_manager.finish_mark not exists for 48-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/canal_json_storage_basic/run.sh using Sink-Type: pulsar... <<================= [Fri Apr 4 15:35:45 CST 2025] <<<<<< run test case canal_json_storage_basic success! >>>>>> table batch_add_table.finish_mark not exists for 2-th check, retry later table processor_delay.t11 exists table processor_delay.t12 not exists for 1-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table ddl_manager.finish_mark not exists for 49-th check, retry later table batch_add_table.finish_mark not exists for 3-th check, retry later table processor_delay.t12 exists table processor_delay.t13 not exists for 1-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/canal_json_storage_partition_table/run.sh using Sink-Type: pulsar... <<================= [Fri Apr 4 15:35:49 CST 2025] <<<<<< run test case canal_json_storage_partition_table success! >>>>>> table ddl_manager.finish_mark not exists for 50-th check, retry later table batch_add_table.finish_mark not exists for 4-th check, retry later Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table processor_delay.t13 exists table processor_delay.t14 not exists for 1-th check, retry later table ddl_manager.finish_mark not exists for 51-th check, retry later table batch_add_table.finish_mark not exists for 5-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table processor_delay.t14 exists table processor_delay.t15 not exists for 1-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/multi_tables_ddl/run.sh using Sink-Type: pulsar... <<================= * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 24927 failed: Connection refused * Failed to connect to 127.0.0.1 port 24927: Connection refused * Closing connection 0 You are running an older version of MinIO released 4 years ago Update: Run `mc admin update` Attempting encryption of all config, IAM users and policies on MinIO backend Endpoint: http://127.0.0.1:24927 Object API (Amazon S3 compatible): Go: https://docs.min.io/docs/golang-client-quickstart-guide Java: https://docs.min.io/docs/java-client-quickstart-guide Python: https://docs.min.io/docs/python-client-quickstart-guide JavaScript: https://docs.min.io/docs/javascript-client-quickstart-guide .NET: https://docs.min.io/docs/dotnet-client-quickstart-guide table batch_add_table.finish_mark not exists for 6-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489358437513809531 TiDB Cluster ID. tikv_gc_leader_uuid 657fee913fc000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-4ttz-989qr, pid:9008, start at 2025-04-04 15:35:52.351912044 +0800 CST m=+2.233712679 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:37:52.359 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:35:52.319 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:25:52.319 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489358437513809531 TiDB Cluster ID. tikv_gc_leader_uuid 657fee913fc000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-4ttz-989qr, pid:9008, start at 2025-04-04 15:35:52.351912044 +0800 CST m=+2.233712679 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:37:52.359 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:35:52.319 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:25:52.319 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489358440627696604 TiDB Cluster ID. tikv_gc_leader_uuid 657fee91480000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-4ttz-989qr, pid:9070, start at 2025-04-04 15:35:52.38499763 +0800 CST m=+2.202052754 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:37:52.394 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:35:52.352 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:25:52.352 +0800 All versions after safe point can be accessed. (DO NOT EDIT) =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/cli_with_auth/run.sh using Sink-Type: pulsar... <<================= The 1 times to try to start tidb cluster... table ddl_manager.finish_mark not exists for 52-th check, retry later check_changefeed_state http://127.0.0.1:2379 a1a44e7c-c71c-41d0-bae4-ca1291ccbf87 finished null + endpoints=http://127.0.0.1:2379 + changefeed_id=a1a44e7c-c71c-41d0-bae4-ca1291ccbf87 + expected_state=finished + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c a1a44e7c-c71c-41d0-bae4-ca1291ccbf87 -s + info='{ "upstream_id": 7489357966190138613, "namespace": "default", "id": "a1a44e7c-c71c-41d0-bae4-ca1291ccbf87", "state": "finished", "checkpoint_tso": 457114161638014983, "checkpoint_time": "2025-04-04 15:35:42.479", "error": null }' + echo '{ "upstream_id": 7489357966190138613, "namespace": "default", "id": "a1a44e7c-c71c-41d0-bae4-ca1291ccbf87", "state": "finished", "checkpoint_tso": 457114161638014983, "checkpoint_time": "2025-04-04 15:35:42.479", "error": null }' { "upstream_id": 7489357966190138613, "namespace": "default", "id": "a1a44e7c-c71c-41d0-bae4-ca1291ccbf87", "state": "finished", "checkpoint_tso": 457114161638014983, "checkpoint_time": "2025-04-04 15:35:42.479", "error": null } ++ echo '{' '"upstream_id":' 7489357966190138613, '"namespace":' '"default",' '"id":' '"a1a44e7c-c71c-41d0-bae4-ca1291ccbf87",' '"state":' '"finished",' '"checkpoint_tso":' 457114161638014983, '"checkpoint_time":' '"2025-04-04' '15:35:42.479",' '"error":' null '}' ++ jq -r .state + state=finished + [[ ! finished == \f\i\n\i\s\h\e\d ]] ++ echo '{' '"upstream_id":' 7489357966190138613, '"namespace":' '"default",' '"id":' '"a1a44e7c-c71c-41d0-bae4-ca1291ccbf87",' '"state":' '"finished",' '"checkpoint_tso":' 457114161638014983, '"checkpoint_time":' '"2025-04-04' '15:35:42.479",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 4 15:35:49 CST 2025] <<<<<< run test case changefeed_finish success! >>>>>> Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/ddl_sequence/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/ddl_sequence/tiflash/log/error.log arg matches is ArgMatches { args: {"memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/ddl_sequence/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/ddl_sequence/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/ddl_sequence/tiflash/db/proxy"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/event_filter/run.sh using Sink-Type: pulsar... <<================= The 1 times to try to start tidb cluster... table processor_delay.t15 exists table processor_delay.t16 not exists for 1-th check, retry later * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 24927 (#0) > GET / HTTP/1.1 > Host: 127.0.0.1:24927 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 403 Forbidden < Accept-Ranges: bytes < Content-Length: 226 < Content-Security-Policy: block-all-mixed-content < Content-Type: application/xml < Server: MinIO/RELEASE.2020-07-27T18-37-02Z < Vary: Origin < X-Amz-Request-Id: 18330D5890082342 < X-Xss-Protection: 1; mode=block < Date: Fri, 04 Apr 2025 07:35:54 GMT < { [226 bytes data] * Connection #0 to host 127.0.0.1 left intact Bucket 's3://logbucket/' created Exiting on signal: INTERRUPT table ddl_manager.finish_mark not exists for 53-th check, retry later + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_sequence.cli.10462.out cli tso query --pd=http://127.0.0.1:2379 table batch_add_table.finish_mark not exists for 7-th check, retry later table processor_delay.t16 exists table processor_delay.t17 not exists for 1-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/event_filter Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/cli_with_auth Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/common_1/run.sh using Sink-Type: pulsar... <<================= table ddl_manager.finish_mark not exists for 54-th check, retry later \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-3484/tiflow-cdc already exists) + set +x + tso='457114165410267138 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 457114165410267138 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 4 15:35:58 CST 2025] <<<<<< START cdc server in ddl_sequence case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_sequence.1049710499.out server --log-file /tmp/tidb_cdc_test/ddl_sequence/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_sequence/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // timeout table batch_add_table.finish_mark not exists for 8-th check, retry later [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container table processor_delay.t17 exists table processor_delay.t18 not exists for 1-th check, retry later [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/force_replicate_table/run.sh using Sink-Type: pulsar... <<================= The 1 times to try to start tidb cluster... table ddl_manager.finish_mark not exists for 55-th check, retry later table batch_add_table.finish_mark not exists for 9-th check, retry later Verifying downstream PD is started... table processor_delay.t18 exists table processor_delay.t19 not exists for 1-th check, retry later \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-3484/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release [Pipeline] } + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 07:36:01 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [817 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0656fcf7-5d30-4992-b2e9-ec047a01b5a9 {"id":"0656fcf7-5d30-4992-b2e9-ec047a01b5a9","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752158} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffba2840e8 0656fcf7-5d30-4992-b2e9-ec047a01b5a9 /tidb/cdc/default/default/upstream/7489358437513809531 {"id":7489358437513809531,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0656fcf7-5d30-4992-b2e9-ec047a01b5a9 {"id":"0656fcf7-5d30-4992-b2e9-ec047a01b5a9","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752158} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffba2840e8 0656fcf7-5d30-4992-b2e9-ec047a01b5a9 /tidb/cdc/default/default/upstream/7489358437513809531 {"id":7489358437513809531,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0656fcf7-5d30-4992-b2e9-ec047a01b5a9 {"id":"0656fcf7-5d30-4992-b2e9-ec047a01b5a9","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752158} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffba2840e8 0656fcf7-5d30-4992-b2e9-ec047a01b5a9 /tidb/cdc/default/default/upstream/7489358437513809531 {"id":7489358437513809531,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x ++ date + echo '[Fri Apr 4 15:36:01 CST 2025] <<<<<< START pulsar cluster in ddl_sequence case >>>>>>' [Fri Apr 4 15:36:01 CST 2025] <<<<<< START pulsar cluster in ddl_sequence case >>>>>> + workdir=/tmp/tidb_cdc_test/ddl_sequence + cluster_type=normal + cd /tmp/tidb_cdc_test/ddl_sequence + DEFAULT_PULSAR_HOME=/usr/local/pulsar + pulsar_dir=/usr/local/pulsar ++ cat + mtls_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderTls brokerClientTlsEnabled=true brokerClientTrustCertsFilePath=/tmp/tidb_cdc_test/ddl_sequence/ca.cert.pem brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls brokerClientAuthenticationParameters={"tlsCertFile":"/tmp/tidb_cdc_test/ddl_sequence/broker_client.cert.pem","tlsKeyFile":"/tmp/tidb_cdc_test/ddl_sequence/broker_client.key-pk8.pem"} brokerServicePortTls=6651 webServicePortTls=8443 tlsTrustCertsFilePath=/tmp/tidb_cdc_test/ddl_sequence/ca.cert.pem tlsCertificateFilePath=/tmp/tidb_cdc_test/ddl_sequence/server.cert.pem tlsKeyFilePath=/tmp/tidb_cdc_test/ddl_sequence/server.key-pk8.pem tlsRequireTrustedClientCertOnConnect=true tlsAllowInsecureConnection=false tlsCertRefreshCheckDurationSec=300' ++ cat + normal_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' ++ cat + mtls_client_conf=' webServiceUrl=https://localhost:8443/ brokerServiceUrl=pulsar+ssl://localhost:6651/ authPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls authParams=tlsCertFile:/tmp/tidb_cdc_test/ddl_sequence/broker_client.cert.pem,tlsKeyFile:/tmp/tidb_cdc_test/ddl_sequence/broker_client.key-pk8.pem tlsTrustCertsFilePath=/tmp/tidb_cdc_test/ddl_sequence/ca.cert.pem' ++ cat + oauth_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/ authPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 authParams={"privateKey":"/tmp/tidb_cdc_test/ddl_sequence/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"}' ++ cat + oauth_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderToken brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 brokerClientAuthenticationParameters={"privateKey":"file:///tmp/tidb_cdc_test/ddl_sequence/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"} tokenSecretKey=data:;base64,U0poWDM2X0thcFlTeWJCdEpxMzVseFhfQnJyNExSVVJTa203UW1YSkdteThwVUZXOUVJT2NWUVBzeWt6OS1qag==' ++ cat + credential_json=' { "client_id":"1234", "client_secret":"e0KVlA2EiBfjoN13olyZd2kv1KL", "audience":"cdc-api-uri", "issuer_url":"http://localhost:9096", "type": "client_credentials" }' ++ cat + cert_server_conf='[ req ] default_bits = 2048 prompt = no default_md = sha256 distinguished_name = dn [ v3_ext ] authorityKeyIdentifier=keyid,issuer:always basicConstraints=CA:FALSE keyUsage=critical, digitalSignature, keyEncipherment extendedKeyUsage=serverAuth subjectAltName=@alt_names [ dn ] CN = server [ alt_names ] DNS.1 = localhost IP.1 = 127.0.0.1' + echo ' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' + cp /usr/local/pulsar/conf/standalone.conf /tmp/tidb_cdc_test/ddl_sequence/pulsar_standalone.conf + pulsar_port=6650 + '[' normal == mtls ']' + '[' normal == oauth ']' + echo 'no cluster type specified, using default configuration.' no cluster type specified, using default configuration. ++ date + echo '[Fri Apr 4 15:36:01 CST 2025] <<<<<< START pulsar cluster in normal mode in ddl_sequence case >>>>>>' [Fri Apr 4 15:36:01 CST 2025] <<<<<< START pulsar cluster in normal mode in ddl_sequence case >>>>>> + echo 'Waiting for pulsar port to be ready...' Waiting for pulsar port to be ready... + i=0 + nc -z localhost 6650 + /usr/local/pulsar/bin/pulsar standalone --config /tmp/tidb_cdc_test/ddl_sequence/pulsar_standalone.conf -nfw --metadata-dir /tmp/tidb_cdc_test/ddl_sequence/pulsar-metadata --bookkeeper-dir /tmp/tidb_cdc_test/ddl_sequence/pulsar-bookie + i=1 + '[' 1 -gt 20 ']' + sleep 2 [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } table ddl_manager.finish_mark not exists for 56-th check, retry later [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } table batch_add_table.finish_mark not exists for 10-th check, retry later table processor_delay.t19 exists table processor_delay.t20 not exists for 1-th check, retry later Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + nc -z localhost 6650 + i=2 + '[' 2 -gt 20 ']' + sleep 2 Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/force_replicate_table Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... table batch_add_table.finish_mark not exists for 11-th check, retry later table ddl_manager.finish_mark not exists for 57-th check, retry later table processor_delay.t20 exists table processor_delay.t21 not exists for 1-th check, retry later + nc -z localhost 6650 + echo 'Waiting for pulsar namespace to be ready...' Waiting for pulsar namespace to be ready... + i=0 + /usr/local/pulsar/bin/pulsar-admin namespaces list public VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489358498707719756 TiDB Cluster ID. tikv_gc_leader_uuid 657fee9db240014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-b84s-w4c6p, pid:7661, start at 2025-04-04 15:36:05.113970666 +0800 CST m=+1.680708758 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:38:05.125 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:36:05.115 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:26:05.115 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489358498707719756 TiDB Cluster ID. tikv_gc_leader_uuid 657fee9db240014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-b84s-w4c6p, pid:7661, start at 2025-04-04 15:36:05.113970666 +0800 CST m=+1.680708758 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:38:05.125 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:36:05.115 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:26:05.115 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489358500077829700 TiDB Cluster ID. tikv_gc_leader_uuid 657fee9db2c0003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-b84s-w4c6p, pid:7729, start at 2025-04-04 15:36:05.070466638 +0800 CST m=+1.564235405 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:38:05.082 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:36:05.067 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:26:05.067 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/event_filter/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/event_filter/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/event_filter/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/event_filter/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/event_filter/tiflash/db/proxy"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489358500607251201 TiDB Cluster ID. tikv_gc_leader_uuid 657fee9de1c0012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-qg8g-pl0pp, pid:11250, start at 2025-04-04 15:36:05.281891705 +0800 CST m=+1.662580929 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:38:05.293 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:36:05.255 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:26:05.255 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489358500607251201 TiDB Cluster ID. tikv_gc_leader_uuid 657fee9de1c0012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-qg8g-pl0pp, pid:11250, start at 2025-04-04 15:36:05.281891705 +0800 CST m=+1.662580929 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:38:05.293 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:36:05.255 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:26:05.255 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489358490910756185 TiDB Cluster ID. tikv_gc_leader_uuid 657fee9dd9c000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-qg8g-pl0pp, pid:11316, start at 2025-04-04 15:36:05.244320169 +0800 CST m=+1.545397248 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:38:05.252 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:36:05.223 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:26:05.223 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/cli_with_auth/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/cli_with_auth/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/cli_with_auth/tiflash-proxy.toml"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/cli_with_auth/tiflash/db/proxy"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/cli_with_auth/tiflash/log/proxy.log"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Verifying downstream PD is started... Starting Upstream TiKV... table ddl_manager.finish_mark not exists for 58-th check, retry later TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table batch_add_table.finish_mark not exists for 12-th check, retry later table processor_delay.t21 exists table processor_delay.t22 not exists for 1-th check, retry later public/default ++ date + echo '[Fri Apr 4 15:36:07 CST 2025] <<<<<< pulsar is ready >>>>>>' [Fri Apr 4 15:36:07 CST 2025] <<<<<< pulsar is ready >>>>>> + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_sequence.cli.10951.out cli changefeed create --start-ts=457114165410267138 '--sink-uri=pulsar://127.0.0.1:6650/ticdc-ddl-sequence-test-32498?protocol=canal-json&enable-tidb-extension=true' [Fri Apr 4 15:36:07 CST 2025] <<<<<< START cdc server in event_filter case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.event_filter.90439045.out server --log-file /tmp/tidb_cdc_test/event_filter/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/event_filter/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_with_auth.cli.12691.out cli tso query --pd=http://127.0.0.1:2379 table ddl_manager.finish_mark not exists for 59-th check, retry later Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... table batch_add_table.finish_mark not exists for 13-th check, retry later Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Create changefeed successfully! ID: 1fce15cd-4ccf-4952-adaa-ea67340aa627 Info: {"upstream_id":7489358437513809531,"namespace":"default","id":"1fce15cd-4ccf-4952-adaa-ea67340aa627","sink_uri":"pulsar://127.0.0.1:6650/ticdc-ddl-sequence-test-32498?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2025-04-04T15:36:08.980728558+08:00","start_ts":457114165410267138,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"pulsar_config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-10-ga306f4894","resolved_ts":457114165410267138,"checkpoint_ts":457114165410267138,"checkpoint_time":"2025-04-04 15:35:56.869"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... table processor_delay.t22 exists table processor_delay.t23 not exists for 1-th check, retry later + set +x + tso='457114168434622465 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 457114168434622465 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x table ddl_manager.finish_mark not exists for 60-th check, retry later table batch_add_table.finish_mark not exists for 14-th check, retry later + set +x [Fri Apr 4 15:36:10 CST 2025] <<<<<< START Pulsar consumer in ddl_sequence case >>>>>> VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489358522596723319 TiDB Cluster ID. tikv_gc_leader_uuid 657feea2b78000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-2xgw-gnw10, pid:4805, start at 2025-04-04 15:36:10.24680358 +0800 CST m=+1.705672054 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:38:10.255 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:36:10.256 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:26:10.256 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489358522596723319 TiDB Cluster ID. tikv_gc_leader_uuid 657feea2b78000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-2xgw-gnw10, pid:4805, start at 2025-04-04 15:36:10.24680358 +0800 CST m=+1.705672054 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:38:10.255 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:36:10.256 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:26:10.256 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489358524707697033 TiDB Cluster ID. tikv_gc_leader_uuid 657feea2b88000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-2xgw-gnw10, pid:4886, start at 2025-04-04 15:36:10.226557329 +0800 CST m=+1.604976848 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:38:10.238 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:36:10.210 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:26:10.210 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/force_replicate_table/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/force_replicate_table/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/force_replicate_table/tiflash/log/proxy.log"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/force_replicate_table/tiflash/db/proxy"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/force_replicate_table/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [Fri Apr 4 15:36:10 CST 2025] <<<<<< START cdc server in cli_with_auth case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_with_auth.1273812740.out server --log-file /tmp/tidb_cdc_test/cli_with_auth/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/cli_with_auth/cdc_data --cluster-id default --config /tmp/tidb_cdc_test/cli_with_auth/server.toml + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 07:36:10 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [817 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1182aa68-048a-459f-b5af-4ca54f220231 {"id":"1182aa68-048a-459f-b5af-4ca54f220231","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752168} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffba584cce 1182aa68-048a-459f-b5af-4ca54f220231 /tidb/cdc/default/default/upstream/7489358498707719756 {"id":7489358498707719756,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1182aa68-048a-459f-b5af-4ca54f220231 {"id":"1182aa68-048a-459f-b5af-4ca54f220231","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752168} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffba584cce 1182aa68-048a-459f-b5af-4ca54f220231 /tidb/cdc/default/default/upstream/7489358498707719756 {"id":7489358498707719756,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1182aa68-048a-459f-b5af-4ca54f220231 {"id":"1182aa68-048a-459f-b5af-4ca54f220231","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752168} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffba584cce 1182aa68-048a-459f-b5af-4ca54f220231 /tidb/cdc/default/default/upstream/7489358498707719756 {"id":7489358498707719756,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x ++ date + echo '[Fri Apr 4 15:36:10 CST 2025] <<<<<< START pulsar cluster in event_filter case >>>>>>' [Fri Apr 4 15:36:10 CST 2025] <<<<<< START pulsar cluster in event_filter case >>>>>> + workdir=/tmp/tidb_cdc_test/event_filter + cluster_type=normal + cd /tmp/tidb_cdc_test/event_filter + DEFAULT_PULSAR_HOME=/usr/local/pulsar + pulsar_dir=/usr/local/pulsar ++ cat + mtls_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderTls brokerClientTlsEnabled=true brokerClientTrustCertsFilePath=/tmp/tidb_cdc_test/event_filter/ca.cert.pem brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls brokerClientAuthenticationParameters={"tlsCertFile":"/tmp/tidb_cdc_test/event_filter/broker_client.cert.pem","tlsKeyFile":"/tmp/tidb_cdc_test/event_filter/broker_client.key-pk8.pem"} brokerServicePortTls=6651 webServicePortTls=8443 tlsTrustCertsFilePath=/tmp/tidb_cdc_test/event_filter/ca.cert.pem tlsCertificateFilePath=/tmp/tidb_cdc_test/event_filter/server.cert.pem tlsKeyFilePath=/tmp/tidb_cdc_test/event_filter/server.key-pk8.pem tlsRequireTrustedClientCertOnConnect=true tlsAllowInsecureConnection=false tlsCertRefreshCheckDurationSec=300' ++ cat + normal_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' ++ cat + mtls_client_conf=' webServiceUrl=https://localhost:8443/ brokerServiceUrl=pulsar+ssl://localhost:6651/ authPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls authParams=tlsCertFile:/tmp/tidb_cdc_test/event_filter/broker_client.cert.pem,tlsKeyFile:/tmp/tidb_cdc_test/event_filter/broker_client.key-pk8.pem tlsTrustCertsFilePath=/tmp/tidb_cdc_test/event_filter/ca.cert.pem' ++ cat + oauth_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/ authPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 authParams={"privateKey":"/tmp/tidb_cdc_test/event_filter/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"}' ++ cat + oauth_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderToken brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 brokerClientAuthenticationParameters={"privateKey":"file:///tmp/tidb_cdc_test/event_filter/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"} tokenSecretKey=data:;base64,U0poWDM2X0thcFlTeWJCdEpxMzVseFhfQnJyNExSVVJTa203UW1YSkdteThwVUZXOUVJT2NWUVBzeWt6OS1qag==' ++ cat + credential_json=' { "client_id":"1234", "client_secret":"e0KVlA2EiBfjoN13olyZd2kv1KL", "audience":"cdc-api-uri", "issuer_url":"http://localhost:9096", "type": "client_credentials" }' ++ cat + cert_server_conf='[ req ] default_bits = 2048 prompt = no default_md = sha256 distinguished_name = dn [ v3_ext ] authorityKeyIdentifier=keyid,issuer:always basicConstraints=CA:FALSE keyUsage=critical, digitalSignature, keyEncipherment extendedKeyUsage=serverAuth subjectAltName=@alt_names [ dn ] CN = server [ alt_names ] DNS.1 = localhost IP.1 = 127.0.0.1' + echo ' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' + cp /usr/local/pulsar/conf/standalone.conf /tmp/tidb_cdc_test/event_filter/pulsar_standalone.conf + pulsar_port=6650 + '[' normal == mtls ']' + '[' normal == oauth ']' + echo 'no cluster type specified, using default configuration.' no cluster type specified, using default configuration. ++ date + echo '[Fri Apr 4 15:36:10 CST 2025] <<<<<< START pulsar cluster in normal mode in event_filter case >>>>>>' [Fri Apr 4 15:36:10 CST 2025] <<<<<< START pulsar cluster in normal mode in event_filter case >>>>>> + echo 'Waiting for pulsar port to be ready...' Waiting for pulsar port to be ready... + i=0 + nc -z localhost 6650 + /usr/local/pulsar/bin/pulsar standalone --config /tmp/tidb_cdc_test/event_filter/pulsar_standalone.conf -nfw --metadata-dir /tmp/tidb_cdc_test/event_filter/pulsar-metadata --bookkeeper-dir /tmp/tidb_cdc_test/event_filter/pulsar-bookie + i=1 + '[' 1 -gt 20 ']' + sleep 2 table processor_delay.t23 exists table processor_delay.t24 not exists for 1-th check, retry later table ddl_manager.finish_mark not exists for 61-th check, retry later table batch_add_table.finish_mark not exists for 15-th check, retry later + nc -z localhost 6650 + i=2 + '[' 2 -gt 20 ']' + sleep 2 table processor_delay.t24 exists table processor_delay.t25 not exists for 1-th check, retry later [Fri Apr 4 15:36:13 CST 2025] <<<<<< START cdc server in force_replicate_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.force_replicate_table.63016303.out server --log-file /tmp/tidb_cdc_test/force_replicate_table/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/force_replicate_table/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 07:36:14 GMT < Content-Length: 861 < Content-Type: text/plain; charset=utf-8 < { [861 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/50b6ff5e-fe61-4a7b-9aad-c6c8d6b7bc4c {"id":"50b6ff5e-fe61-4a7b-9aad-c6c8d6b7bc4c","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752171} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffba5cec54 50b6ff5e-fe61-4a7b-9aad-c6c8d6b7bc4c /tidb/cdc/default/default/upstream/7489358500607251201 {"id":7489358500607251201,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/50b6ff5e-fe61-4a7b-9aad-c6c8d6b7bc4c {"id":"50b6ff5e-fe61-4a7b-9aad-c6c8d6b7bc4c","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752171} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffba5cec54 50b6ff5e-fe61-4a7b-9aad-c6c8d6b7bc4c /tidb/cdc/default/default/upstream/7489358500607251201 {"id":7489358500607251201,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/50b6ff5e-fe61-4a7b-9aad-c6c8d6b7bc4c {"id":"50b6ff5e-fe61-4a7b-9aad-c6c8d6b7bc4c","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752171} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffba5cec54 50b6ff5e-fe61-4a7b-9aad-c6c8d6b7bc4c /tidb/cdc/default/default/upstream/7489358500607251201 {"id":7489358500607251201,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x ++ date + echo '[Fri Apr 4 15:36:14 CST 2025] <<<<<< START pulsar cluster in cli_with_auth case >>>>>>' [Fri Apr 4 15:36:14 CST 2025] <<<<<< START pulsar cluster in cli_with_auth case >>>>>> + workdir=/tmp/tidb_cdc_test/cli_with_auth + cluster_type=normal + cd /tmp/tidb_cdc_test/cli_with_auth + DEFAULT_PULSAR_HOME=/usr/local/pulsar + pulsar_dir=/usr/local/pulsar ++ cat + mtls_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderTls brokerClientTlsEnabled=true brokerClientTrustCertsFilePath=/tmp/tidb_cdc_test/cli_with_auth/ca.cert.pem brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls brokerClientAuthenticationParameters={"tlsCertFile":"/tmp/tidb_cdc_test/cli_with_auth/broker_client.cert.pem","tlsKeyFile":"/tmp/tidb_cdc_test/cli_with_auth/broker_client.key-pk8.pem"} brokerServicePortTls=6651 webServicePortTls=8443 tlsTrustCertsFilePath=/tmp/tidb_cdc_test/cli_with_auth/ca.cert.pem tlsCertificateFilePath=/tmp/tidb_cdc_test/cli_with_auth/server.cert.pem tlsKeyFilePath=/tmp/tidb_cdc_test/cli_with_auth/server.key-pk8.pem tlsRequireTrustedClientCertOnConnect=true tlsAllowInsecureConnection=false tlsCertRefreshCheckDurationSec=300' ++ cat + normal_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' ++ cat + mtls_client_conf=' webServiceUrl=https://localhost:8443/ brokerServiceUrl=pulsar+ssl://localhost:6651/ authPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls authParams=tlsCertFile:/tmp/tidb_cdc_test/cli_with_auth/broker_client.cert.pem,tlsKeyFile:/tmp/tidb_cdc_test/cli_with_auth/broker_client.key-pk8.pem tlsTrustCertsFilePath=/tmp/tidb_cdc_test/cli_with_auth/ca.cert.pem' ++ cat + oauth_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/ authPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 authParams={"privateKey":"/tmp/tidb_cdc_test/cli_with_auth/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"}' ++ cat + oauth_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderToken brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 brokerClientAuthenticationParameters={"privateKey":"file:///tmp/tidb_cdc_test/cli_with_auth/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"} tokenSecretKey=data:;base64,U0poWDM2X0thcFlTeWJCdEpxMzVseFhfQnJyNExSVVJTa203UW1YSkdteThwVUZXOUVJT2NWUVBzeWt6OS1qag==' ++ cat + credential_json=' { "client_id":"1234", "client_secret":"e0KVlA2EiBfjoN13olyZd2kv1KL", "audience":"cdc-api-uri", "issuer_url":"http://localhost:9096", "type": "client_credentials" }' ++ cat + cert_server_conf='[ req ] default_bits = 2048 prompt = no default_md = sha256 distinguished_name = dn [ v3_ext ] authorityKeyIdentifier=keyid,issuer:always basicConstraints=CA:FALSE keyUsage=critical, digitalSignature, keyEncipherment extendedKeyUsage=serverAuth subjectAltName=@alt_names [ dn ] CN = server [ alt_names ] DNS.1 = localhost IP.1 = 127.0.0.1' + echo ' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' + cp /usr/local/pulsar/conf/standalone.conf /tmp/tidb_cdc_test/cli_with_auth/pulsar_standalone.conf + pulsar_port=6650 + '[' normal == mtls ']' + '[' normal == oauth ']' + echo 'no cluster type specified, using default configuration.' no cluster type specified, using default configuration. ++ date + echo '[Fri Apr 4 15:36:14 CST 2025] <<<<<< START pulsar cluster in normal mode in cli_with_auth case >>>>>>' [Fri Apr 4 15:36:14 CST 2025] <<<<<< START pulsar cluster in normal mode in cli_with_auth case >>>>>> + echo 'Waiting for pulsar port to be ready...' Waiting for pulsar port to be ready... + i=0 + nc -z localhost 6650 + /usr/local/pulsar/bin/pulsar standalone --config /tmp/tidb_cdc_test/cli_with_auth/pulsar_standalone.conf -nfw --metadata-dir /tmp/tidb_cdc_test/cli_with_auth/pulsar-metadata --bookkeeper-dir /tmp/tidb_cdc_test/cli_with_auth/pulsar-bookie + i=1 + '[' 1 -gt 20 ']' + sleep 2 table ddl_manager.finish_mark not exists for 62-th check, retry later table batch_add_table.finish_mark not exists for 16-th check, retry later table processor_delay.t25 exists table processor_delay.t26 not exists for 1-th check, retry later + nc -z localhost 6650 + echo 'Waiting for pulsar namespace to be ready...' Waiting for pulsar namespace to be ready... + i=0 + /usr/local/pulsar/bin/pulsar-admin namespaces list public + nc -z localhost 6650 + i=2 + '[' 2 -gt 20 ']' + sleep 2 table ddl_manager.finish_mark not exists for 63-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 07:36:16 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [817 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/652996b5-52db-4d0e-87b7-df712436f2d5 {"id":"652996b5-52db-4d0e-87b7-df712436f2d5","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752173} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffba7048cf 652996b5-52db-4d0e-87b7-df712436f2d5 /tidb/cdc/default/default/upstream/7489358522596723319 {"id":7489358522596723319,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/652996b5-52db-4d0e-87b7-df712436f2d5 {"id":"652996b5-52db-4d0e-87b7-df712436f2d5","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752173} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffba7048cf 652996b5-52db-4d0e-87b7-df712436f2d5 /tidb/cdc/default/default/upstream/7489358522596723319 {"id":7489358522596723319,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/652996b5-52db-4d0e-87b7-df712436f2d5 {"id":"652996b5-52db-4d0e-87b7-df712436f2d5","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752173} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffba7048cf 652996b5-52db-4d0e-87b7-df712436f2d5 /tidb/cdc/default/default/upstream/7489358522596723319 {"id":7489358522596723319,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x ++ date + echo '[Fri Apr 4 15:36:16 CST 2025] <<<<<< START pulsar cluster in force_replicate_table case >>>>>>' [Fri Apr 4 15:36:16 CST 2025] <<<<<< START pulsar cluster in force_replicate_table case >>>>>> + workdir=/tmp/tidb_cdc_test/force_replicate_table + cluster_type=normal + cd /tmp/tidb_cdc_test/force_replicate_table + DEFAULT_PULSAR_HOME=/usr/local/pulsar + pulsar_dir=/usr/local/pulsar ++ cat + mtls_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderTls brokerClientTlsEnabled=true brokerClientTrustCertsFilePath=/tmp/tidb_cdc_test/force_replicate_table/ca.cert.pem brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls brokerClientAuthenticationParameters={"tlsCertFile":"/tmp/tidb_cdc_test/force_replicate_table/broker_client.cert.pem","tlsKeyFile":"/tmp/tidb_cdc_test/force_replicate_table/broker_client.key-pk8.pem"} brokerServicePortTls=6651 webServicePortTls=8443 tlsTrustCertsFilePath=/tmp/tidb_cdc_test/force_replicate_table/ca.cert.pem tlsCertificateFilePath=/tmp/tidb_cdc_test/force_replicate_table/server.cert.pem tlsKeyFilePath=/tmp/tidb_cdc_test/force_replicate_table/server.key-pk8.pem tlsRequireTrustedClientCertOnConnect=true tlsAllowInsecureConnection=false tlsCertRefreshCheckDurationSec=300' ++ cat + normal_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' ++ cat + mtls_client_conf=' webServiceUrl=https://localhost:8443/ brokerServiceUrl=pulsar+ssl://localhost:6651/ authPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls authParams=tlsCertFile:/tmp/tidb_cdc_test/force_replicate_table/broker_client.cert.pem,tlsKeyFile:/tmp/tidb_cdc_test/force_replicate_table/broker_client.key-pk8.pem tlsTrustCertsFilePath=/tmp/tidb_cdc_test/force_replicate_table/ca.cert.pem' ++ cat + oauth_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/ authPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 authParams={"privateKey":"/tmp/tidb_cdc_test/force_replicate_table/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"}' ++ cat + oauth_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderToken brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 brokerClientAuthenticationParameters={"privateKey":"file:///tmp/tidb_cdc_test/force_replicate_table/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"} tokenSecretKey=data:;base64,U0poWDM2X0thcFlTeWJCdEpxMzVseFhfQnJyNExSVVJTa203UW1YSkdteThwVUZXOUVJT2NWUVBzeWt6OS1qag==' ++ cat + credential_json=' { "client_id":"1234", "client_secret":"e0KVlA2EiBfjoN13olyZd2kv1KL", "audience":"cdc-api-uri", "issuer_url":"http://localhost:9096", "type": "client_credentials" }' ++ cat + cert_server_conf='[ req ] default_bits = 2048 prompt = no default_md = sha256 distinguished_name = dn [ v3_ext ] authorityKeyIdentifier=keyid,issuer:always basicConstraints=CA:FALSE keyUsage=critical, digitalSignature, keyEncipherment extendedKeyUsage=serverAuth subjectAltName=@alt_names [ dn ] CN = server [ alt_names ] DNS.1 = localhost IP.1 = 127.0.0.1' + echo ' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' + cp /usr/local/pulsar/conf/standalone.conf /tmp/tidb_cdc_test/force_replicate_table/pulsar_standalone.conf + pulsar_port=6650 + '[' normal == mtls ']' + '[' normal == oauth ']' + echo 'no cluster type specified, using default configuration.' no cluster type specified, using default configuration. ++ date + echo '[Fri Apr 4 15:36:16 CST 2025] <<<<<< START pulsar cluster in normal mode in force_replicate_table case >>>>>>' [Fri Apr 4 15:36:16 CST 2025] <<<<<< START pulsar cluster in normal mode in force_replicate_table case >>>>>> + echo 'Waiting for pulsar port to be ready...' Waiting for pulsar port to be ready... + i=0 + nc -z localhost 6650 + /usr/local/pulsar/bin/pulsar standalone --config /tmp/tidb_cdc_test/force_replicate_table/pulsar_standalone.conf -nfw --metadata-dir /tmp/tidb_cdc_test/force_replicate_table/pulsar-metadata --bookkeeper-dir /tmp/tidb_cdc_test/force_replicate_table/pulsar-bookie + i=1 + '[' 1 -gt 20 ']' + sleep 2 table batch_add_table.finish_mark not exists for 17-th check, retry later table processor_delay.t26 exists table processor_delay.t27 not exists for 1-th check, retry later table ddl_sequence.finish_mark not exists for 1-th check, retry later public/default ++ date + echo '[Fri Apr 4 15:36:17 CST 2025] <<<<<< pulsar is ready >>>>>>' [Fri Apr 4 15:36:17 CST 2025] <<<<<< pulsar is ready >>>>>> + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.event_filter.cli.9508.out cli changefeed create '--sink-uri=pulsar://127.0.0.1:6650/ticdc-event-filter-15934?protocol=canal-json&enable-tidb-extension=true' --server=127.0.0.1:8300 --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/event_filter/conf/cf.toml Create changefeed successfully! ID: f734ae88-a548-4481-9e72-7bea5fdaf329 Info: {"upstream_id":7489358498707719756,"namespace":"default","id":"f734ae88-a548-4481-9e72-7bea5fdaf329","sink_uri":"pulsar://127.0.0.1:6650/ticdc-event-filter-15934?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2025-04-04T15:36:18.321622276+08:00","start_ts":457114170874920962,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["event_filter.*"],"event_filters":[{"matcher":["event_filter.t1"],"ignore_event":["drop table","delete"],"ignore_sql":null,"ignore_insert_value_expr":"id = 2 or city = 'tokyo'","ignore_update_new_value_expr":"","ignore_update_old_value_expr":"","ignore_delete_value_expr":""},{"matcher":["event_filter.t_truncate"],"ignore_event":["truncate table"],"ignore_sql":null,"ignore_insert_value_expr":"","ignore_update_new_value_expr":"","ignore_update_old_value_expr":"","ignore_delete_value_expr":""},{"matcher":["event_filter.t_alter"],"ignore_event":["alter table"],"ignore_sql":null,"ignore_insert_value_expr":"","ignore_update_new_value_expr":"","ignore_update_old_value_expr":"","ignore_delete_value_expr":""},{"matcher":["event_filter.t_name*"],"ignore_event":["rename table"],"ignore_sql":null,"ignore_insert_value_expr":"","ignore_update_new_value_expr":"","ignore_update_old_value_expr":"","ignore_delete_value_expr":""}]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"pulsar_config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-10-ga306f4894","resolved_ts":457114170874920962,"checkpoint_ts":457114170874920962,"checkpoint_time":"2025-04-04 15:36:17.715"} PASS table ddl_manager.finish_mark not exists for 64-th check, retry later + nc -z localhost 6650 + i=2 + '[' 2 -gt 20 ']' + sleep 2 coverage: 2.5% of statements in github.com/pingcap/tiflow/... table batch_add_table.finish_mark not exists for 18-th check, retry later + nc -z localhost 6650 + echo 'Waiting for pulsar namespace to be ready...' Waiting for pulsar namespace to be ready... + i=0 + /usr/local/pulsar/bin/pulsar-admin namespaces list public table ddl_sequence.finish_mark not exists for 2-th check, retry later table processor_delay.t27 exists table processor_delay.t28 not exists for 1-th check, retry later + set +x [Fri Apr 4 15:36:19 CST 2025] <<<<<< START Pulsar consumer in event_filter case >>>>>> table event_filter.t1 does not exists table event_filter.t1 not exists for 1-th check, retry later table ddl_manager.finish_mark not exists for 65-th check, retry later [2025/04/04 15:36:12.489 +08:00] [INFO] [pd_service_discovery.go:1025] ["[pd] switch leader"] [new-leader=http://127.0.0.1:2379] [old-leader=] [2025/04/04 15:36:12.489 +08:00] [INFO] [pd_service_discovery.go:499] ["[pd] init cluster id"] [cluster-id=7489357969709569958] [2025/04/04 15:36:12.489 +08:00] [INFO] [client.go:532] ["[pd] changing service mode"] [old-mode=UNKNOWN_SVC_MODE] [new-mode=PD_SVC_MODE] [2025/04/04 15:36:12.489 +08:00] [INFO] [tso_client.go:296] ["[tso] switch dc tso global allocator serving url"] [dc-location=global] [new-url=http://127.0.0.1:2379] [2025/04/04 15:36:12.489 +08:00] [INFO] [client.go:538] ["[pd] service mode changed"] [old-mode=UNKNOWN_SVC_MODE] [new-mode=PD_SVC_MODE] [2025/04/04 15:36:12.489 +08:00] [INFO] [tso_dispatcher.go:140] ["[tso] start tso deadline watcher"] [dc-location=global] [2025/04/04 15:36:12.489 +08:00] [INFO] [tso_client.go:132] ["[tso] start tso dispatcher check loop"] [2025/04/04 15:36:12.489 +08:00] [INFO] [tso_dispatcher.go:198] ["[tso] tso dispatcher created"] [dc-location=global] [2025/04/04 15:36:12.490 +08:00] [INFO] [tso_dispatcher.go:723] ["[tso] switching tso rpc concurrency"] [old=0] [new=1] [2025/04/04 15:36:12.490 +08:00] [INFO] [tso_dispatcher.go:476] ["[tso] start tso connection contexts updater"] [dc-location=global] [2025/04/04 15:36:12.491 +08:00] [INFO] [pd_service_discovery.go:1025] ["[pd] switch leader"] [new-leader=http://127.0.0.1:2379] [old-leader=] [2025/04/04 15:36:12.491 +08:00] [INFO] [pd_service_discovery.go:499] ["[pd] init cluster id"] [cluster-id=7489357969709569958] [2025/04/04 15:36:12.491 +08:00] [INFO] [client.go:532] ["[pd] changing service mode"] [old-mode=UNKNOWN_SVC_MODE] [new-mode=PD_SVC_MODE] [2025/04/04 15:36:12.491 +08:00] [INFO] [tso_client.go:296] ["[tso] switch dc tso global allocator serving url"] [dc-location=global] [new-url=http://127.0.0.1:2379] [2025/04/04 15:36:12.491 +08:00] [INFO] [client.go:538] ["[pd] service mode changed"] [old-mode=UNKNOWN_SVC_MODE] [new-mode=PD_SVC_MODE] [2025/04/04 15:36:12.491 +08:00] [INFO] [tso_dispatcher.go:140] ["[tso] start tso deadline watcher"] [dc-location=global] [2025/04/04 15:36:12.491 +08:00] [INFO] [tso_dispatcher.go:198] ["[tso] tso dispatcher created"] [dc-location=global] [2025/04/04 15:36:12.491 +08:00] [INFO] [tso_client.go:132] ["[tso] start tso dispatcher check loop"] [2025/04/04 15:36:12.491 +08:00] [INFO] [tso_dispatcher.go:723] ["[tso] switching tso rpc concurrency"] [old=0] [new=1] [2025/04/04 15:36:12.491 +08:00] [INFO] [tso_dispatcher.go:476] ["[tso] start tso connection contexts updater"] [dc-location=global] [2025/04/04 15:36:12.492 +08:00] [INFO] [tikv_driver.go:201] ["using API V1."] [2025/04/04 15:36:12.492 +08:00] [INFO] [main.go:180] ["genLock started"] [2025/04/04 15:36:12.493 +08:00] [INFO] [store_cache.go:532] ["change store resolve state"] [store=1] [addr=127.0.0.1:20162] [from=unresolved] [to=resolved] [liveness-state=reachable] [2025/04/04 15:36:14.492 +08:00] [INFO] [pd.go:441] ["adaptive update ts interval state transition"] [configuredInterval=2s] [prevAdaptiveUpdateInterval=2s] [newAdaptiveUpdateInterval=2s] [requiredStaleness=0s] [prevState=unknown(0)] [newState=normal] + nc -z localhost 6650 + echo 'Waiting for pulsar namespace to be ready...' Waiting for pulsar namespace to be ready... + i=0 + /usr/local/pulsar/bin/pulsar-admin namespaces list public table batch_add_table.finish_mark not exists for 19-th check, retry later public/default ++ date + echo '[Fri Apr 4 15:36:20 CST 2025] <<<<<< pulsar is ready >>>>>>' [Fri Apr 4 15:36:20 CST 2025] <<<<<< pulsar is ready >>>>>> + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_with_auth.cli.13209.out cli changefeed create --start-ts=457114168434622465 '--sink-uri=pulsar://127.0.0.1:6650/ticdc-cli-test-26121?protocol=canal-json&enable-tidb-extension=true' --tz=Asia/Shanghai -c=custom-changefeed-name [WARN] --tz is deprecated in changefeed settings. table ddl_sequence.finish_mark not exists for 3-th check, retry later Create changefeed successfully! ID: custom-changefeed-name Info: {"upstream_id":7489358500607251201,"namespace":"default","id":"custom-changefeed-name","sink_uri":"pulsar://127.0.0.1:6650/ticdc-cli-test-26121?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2025-04-04T15:36:21.36180798+08:00","start_ts":457114168434622465,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"pulsar_config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-10-ga306f4894","resolved_ts":457114168434622465,"checkpoint_ts":457114168434622465,"checkpoint_time":"2025-04-04 15:36:08.406"} PASS table processor_delay.t28 exists table processor_delay.t29 not exists for 1-th check, retry later coverage: 2.4% of statements in github.com/pingcap/tiflow/... table event_filter.t1 exists table event_filter.t_normal not exists for 1-th check, retry later table ddl_manager.finish_mark not exists for 66-th check, retry later Tenant not found Reason: Tenant not found table batch_add_table.finish_mark not exists for 20-th check, retry later + i=1 + '[' 1 -gt 20 ']' + sleep 2 table ddl_sequence.finish_mark not exists for 4-th check, retry later [2025/04/04 15:36:22.494 +08:00] [INFO] [store_cache.go:532] ["change store resolve state"] [store=4] [addr=127.0.0.1:20160] [from=unresolved] [to=resolved] [liveness-state=reachable] [2025/04/04 15:36:22.494 +08:00] [INFO] [store_cache.go:532] ["change store resolve state"] [store=5] [addr=127.0.0.1:20161] [from=unresolved] [to=resolved] [liveness-state=reachable] [2025/04/04 15:36:22.494 +08:00] [INFO] [store_cache.go:532] ["change store resolve state"] [store=148] [addr=127.0.0.1:9500] [from=unresolved] [to=resolved] [liveness-state=reachable] [2025/04/04 15:36:22.501 +08:00] [INFO] [main.go:196] ["genLock done"] [2025/04/04 15:36:22.501 +08:00] [INFO] [tso_dispatcher.go:264] ["[tso] stop fetching the pending tso requests due to context canceled"] [dc-location=global] [2025/04/04 15:36:22.501 +08:00] [INFO] [tso_client.go:147] ["[tso] exit tso dispatcher check loop"] [2025/04/04 15:36:22.501 +08:00] [INFO] [tso_dispatcher.go:158] ["[tso] exit tso deadline watcher"] [dc-location=global] [2025/04/04 15:36:22.501 +08:00] [INFO] [tso_dispatcher.go:201] ["[tso] exit tso dispatcher"] [dc-location=global] [2025/04/04 15:36:22.501 +08:00] [INFO] [resource_manager_client.go:296] ["[resource manager] exit resource token dispatcher"] [2025/04/04 15:36:22.501 +08:00] [INFO] [tso_dispatcher.go:490] ["[tso] exit tso connection contexts updater"] [dc-location=global] [2025/04/04 15:36:22.501 +08:00] [INFO] [pd_service_discovery.go:551] ["[pd] exit member loop due to context canceled"] [2025/04/04 15:36:22.501 +08:00] [INFO] [tso_client.go:157] ["[tso] closing tso client"] [2025/04/04 15:36:22.501 +08:00] [INFO] [tso_client.go:162] ["[tso] close tso client"] [2025/04/04 15:36:22.501 +08:00] [INFO] [tso_stream.go:359] ["tsoStream.recvLoop ended"] [stream=127.0.0.1:2379-1] [error="rpc error: code = Canceled desc = context canceled"] [errorVerbose="rpc error: code = Canceled desc = context canceled\ngithub.com/tikv/pd/client.(*tsoStream).recvLoop\n\t/go/pkg/mod/github.com/tikv/pd/client@v0.0.0-20241111073742-238d4d79ea31/tso_stream.go:427\nruntime.goexit\n\t/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.23.4.linux-amd64/src/runtime/asm_amd64.s:1700"] [2025/04/04 15:36:22.501 +08:00] [INFO] [tso_client.go:164] ["[tso] tso client is closed"] [2025/04/04 15:36:22.501 +08:00] [INFO] [pd_service_discovery.go:644] ["[pd] close pd service discovery client"] + set +x [Fri Apr 4 15:36:22 CST 2025] <<<<<< START Pulsar consumer in cli_with_auth case >>>>>> table test.simple not exists for 1-th check, retry later table processor_delay.t29 exists table processor_delay.t30 not exists for 1-th check, retry later table ddl_manager.finish_mark not exists for 67-th check, retry later table event_filter.t_normal exists table event_filter.t_truncate not exists for 1-th check, retry later table ddl_sequence.finish_mark exists table batch_add_table.finish_mark not exists for 21-th check, retry later check diff successfully table test.simple exists table test.`simple-dash` not exists for 1-th check, retry later + /usr/local/pulsar/bin/pulsar-admin namespaces list public table processor_delay.t30 exists table processor_delay.t31 not exists for 1-th check, retry later wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... table event_filter.t_truncate exists table event_filter.t_alter not exists for 1-th check, retry later cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 4 15:36:26 CST 2025] <<<<<< run test case ddl_sequence success! >>>>>> table ddl_manager.finish_mark not exists for 68-th check, retry later table batch_add_table.finish_mark not exists for 22-th check, retry later public/default ++ date + echo '[Fri Apr 4 15:36:26 CST 2025] <<<<<< pulsar is ready >>>>>>' [Fri Apr 4 15:36:26 CST 2025] <<<<<< pulsar is ready >>>>>> table test.`simple-dash` exists + endpoints=http://127.0.0.1:2379 + changefeed_id=custom-changefeed-name + expected_state=normal + error_msg=null + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c custom-changefeed-name -s + info='{ "upstream_id": 7489358500607251201, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 457114168958648340, "checkpoint_time": "2025-04-04 15:36:10.405", "error": null }' + echo '{ "upstream_id": 7489358500607251201, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 457114168958648340, "checkpoint_time": "2025-04-04 15:36:10.405", "error": null }' { "upstream_id": 7489358500607251201, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 457114168958648340, "checkpoint_time": "2025-04-04 15:36:10.405", "error": null } ++ echo '{' '"upstream_id":' 7489358500607251201, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 457114168958648340, '"checkpoint_time":' '"2025-04-04' '15:36:10.405",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7489358500607251201, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 457114168958648340, '"checkpoint_time":' '"2025-04-04' '15:36:10.405",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] changefeed count 1 check pass, pd_addr: http://127.0.0.1:2379 changefeed count 1 check pass, pd_addr: http://127.0.0.1:2679 table processor_delay.t31 exists table processor_delay.t32 not exists for 1-th check, retry later changefeed count 1 check pass, pd_addr: http://127.0.0.1:2779 changefeed count 1 check pass, pd_addr: http://127.0.0.1:2379,http://127.0.0.1:2679,http://127.0.0.1:2779 Create changefeed successfully! ID: b5d80de8-201d-404e-8e1f-f0e42d2e9346 Info: {"upstream_id":7489358522596723319,"namespace":"default","id":"b5d80de8-201d-404e-8e1f-f0e42d2e9346","sink_uri":"pulsar://127.0.0.1:6650/ticdc-force_replicate_table-test-21861?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2025-04-04T15:36:27.580945681+08:00","start_ts":457114169653592065,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":true,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"pulsar_config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-10-ga306f4894","resolved_ts":457114169653592065,"checkpoint_ts":457114169653592065,"checkpoint_time":"2025-04-04 15:36:13.056"} [Fri Apr 4 15:36:27 CST 2025] <<<<<< START Pulsar consumer in force_replicate_table case >>>>>> table test.t2 not exists for 1-th check, retry later table force_replicate_table.t0 not exists for 1-th check, retry later table event_filter.t_alter exists table event_filter.t_name not exists for 1-th check, retry later table ddl_manager.finish_mark not exists for 69-th check, retry later Error: [CDC:ErrChangefeedUpdateRefused]changefeed update error: can only update changefeed config when it is stopped or failed update changefeed config should fail when changefeed is running, got Diff of changefeed config: {Type:update Path:[Config CaseSensitive] From:false To:true} {Type:update Path:[Config SyncPointInterval] From: To:0xc003982378} {Type:update Path:[Config SyncPointRetention] From: To:0xc003982390} {Type:update Path:[Config Consistent] From: To:0xc000a46ee0} {Type:update Path:[Config Scheduler EnableTableAcrossNodes] From:false To:true} + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_with_auth.cli.13521.out cli changefeed --changefeed-id custom-changefeed-name pause table batch_add_table.finish_mark not exists for 23-th check, retry later PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... table processor_delay.t32 exists table processor_delay.t33 not exists for 1-th check, retry later table event_filter.t_name exists table event_filter.t_name1 not exists for 1-th check, retry later table ddl_manager.finish_mark not exists for 70-th check, retry later + set +x table batch_add_table.finish_mark exists table test.t_1 does not exists table test.t2 not exists for 2-th check, retry later table force_replicate_table.t0 exists table force_replicate_table.t1 not exists for 1-th check, retry later check diff successfully wait process cdc.test exit for 1-th time... table processor_delay.t33 exists table processor_delay.t34 not exists for 1-th check, retry later wait process cdc.test exit for 2-th time... table ddl_manager.finish_mark not exists for 71-th check, retry later cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 4 15:36:32 CST 2025] <<<<<< run test case batch_add_table success! >>>>>> table test.t2 not exists for 3-th check, retry later table force_replicate_table.t1 exists table force_replicate_table.t2 not exists for 1-th check, retry later table event_filter.t_name1 exists table event_filter.t_name2 not exists for 1-th check, retry later table processor_delay.t34 exists table processor_delay.t35 not exists for 1-th check, retry later + endpoints=http://127.0.0.1:2379 + changefeed_id=custom-changefeed-name + expected_state=stopped + error_msg=null + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c custom-changefeed-name -s + info='{ "upstream_id": 7489358500607251201, "namespace": "default", "id": "custom-changefeed-name", "state": "stopped", "checkpoint_tso": 457114173480894466, "checkpoint_time": "2025-04-04 15:36:27.656", "error": null }' + echo '{ "upstream_id": 7489358500607251201, "namespace": "default", "id": "custom-changefeed-name", "state": "stopped", "checkpoint_tso": 457114173480894466, "checkpoint_time": "2025-04-04 15:36:27.656", "error": null }' { "upstream_id": 7489358500607251201, "namespace": "default", "id": "custom-changefeed-name", "state": "stopped", "checkpoint_tso": 457114173480894466, "checkpoint_time": "2025-04-04 15:36:27.656", "error": null } ++ echo '{' '"upstream_id":' 7489358500607251201, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"stopped",' '"checkpoint_tso":' 457114173480894466, '"checkpoint_time":' '"2025-04-04' '15:36:27.656",' '"error":' null '}' ++ jq -r .state + state=stopped + [[ ! stopped == \s\t\o\p\p\e\d ]] ++ echo '{' '"upstream_id":' 7489358500607251201, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"stopped",' '"checkpoint_tso":' 457114173480894466, '"checkpoint_time":' '"2025-04-04' '15:36:27.656",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_with_auth.cli.13595.out cli changefeed update --pd=http://127.0.0.1:2379,http://127.0.0.1:2679,http://127.0.0.1:2779 --config=/tmp/tidb_cdc_test/cli_with_auth/changefeed.toml --no-confirm --changefeed-id custom-changefeed-name Diff of changefeed config: {Type:update Path:[Config CaseSensitive] From:false To:true} {Type:update Path:[Config SyncPointInterval] From: To:0xc003d3b798} {Type:update Path:[Config SyncPointRetention] From: To:0xc003d3b7a8} {Type:update Path:[Config Consistent] From: To:0xc000f4a7e0} {Type:update Path:[Config Scheduler EnableTableAcrossNodes] From:false To:true} Update changefeed config successfully! ID: custom-changefeed-name Info: {"upstream_id":7489358500607251201,"namespace":"default","id":"custom-changefeed-name","sink_uri":"pulsar://127.0.0.1:6650/ticdc-cli-test-26121?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2025-04-04T15:36:21.36180798+08:00","start_ts":457114168434622465,"admin_job_type":1,"config":{"memory_quota":1073741824,"case_sensitive":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_table_monitor":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","encoder_concurrency":32,"terminator":"\r\n","enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"pulsar_config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"stopped","creator_version":"v9.0.0-beta.1-10-ga306f4894","resolved_ts":0,"checkpoint_ts":457114173480894466,"checkpoint_time":"2025-04-04 15:36:27.656"} PASS table ddl_manager.finish_mark not exists for 72-th check, retry later coverage: 2.9% of statements in github.com/pingcap/tiflow/... table test.t2 not exists for 4-th check, retry later table force_replicate_table.t2 exists table force_replicate_table.t3 not exists for 1-th check, retry later table event_filter.t_name2 exists table event_filter.t_name3 not exists for 1-th check, retry later table processor_delay.t35 exists table processor_delay.t36 not exists for 1-th check, retry later + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_with_auth.cli.13629.out cli changefeed --changefeed-id custom-changefeed-name resume PASS coverage: 2.1% of statements in github.com/pingcap/tiflow/... table test.t2 not exists for 5-th check, retry later table force_replicate_table.t3 exists table force_replicate_table.t4 not exists for 1-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/resourcecontrol/run.sh using Sink-Type: pulsar... <<================= The 1 times to try to start tidb cluster... table event_filter.t_name3 exists table ddl_manager.finish_mark not exists for 73-th check, retry later table event_filter.finish_mark exists check diff failed 1-th time, retry later + set +x table processor_delay.t36 exists table processor_delay.t37 not exists for 1-th check, retry later table test.t2 not exists for 6-th check, retry later table ddl_manager.finish_mark not exists for 74-th check, retry later table force_replicate_table.t4 exists table force_replicate_table.t5 not exists for 1-th check, retry later table processor_delay.t37 exists table processor_delay.t38 not exists for 1-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/resourcecontrol Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... check diff failed 2-th time, retry later table test.t2 exists check diff successfully table ddl_manager.finish_mark not exists for 75-th check, retry later table force_replicate_table.t5 exists table force_replicate_table.t6 not exists for 1-th check, retry later + endpoints=http://127.0.0.1:2379 + changefeed_id=custom-changefeed-name + expected_state=normal + error_msg=null + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c custom-changefeed-name -s + info='{ "upstream_id": 7489358500607251201, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 457114176626360325, "checkpoint_time": "2025-04-04 15:36:39.655", "error": null }' + echo '{ "upstream_id": 7489358500607251201, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 457114176626360325, "checkpoint_time": "2025-04-04 15:36:39.655", "error": null }' { "upstream_id": 7489358500607251201, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 457114176626360325, "checkpoint_time": "2025-04-04 15:36:39.655", "error": null } ++ echo '{' '"upstream_id":' 7489358500607251201, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 457114176626360325, '"checkpoint_time":' '"2025-04-04' '15:36:39.655",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7489358500607251201, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 457114176626360325, '"checkpoint_time":' '"2025-04-04' '15:36:39.655",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_with_auth.cli.13708.out cli changefeed --changefeed-id custom-changefeed-name remove wait process cdc.test exit for 1-th time... Changefeed remove successfully. ID: custom-changefeed-name CheckpointTs: 457114176626360325 SinkURI: pulsar://127.0.0.1:6650/ticdc-cli-test-26121?protocol=canal-json&enable-tidb-extension=true PASS wait process cdc.test exit for 2-th time... table processor_delay.t38 exists table processor_delay.t39 not exists for 1-th check, retry later coverage: 2.0% of statements in github.com/pingcap/tiflow/... check diff successfully cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 4 15:36:41 CST 2025] <<<<<< run test case resolve_lock success! >>>>>> wait process cdc.test exit for 1-th time... table ddl_manager.finish_mark not exists for 76-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/multi_rocks/run.sh using Sink-Type: pulsar... <<================= wait process cdc.test exit for 2-th time... table force_replicate_table.t6 not exists for 2-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + set +x cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 4 15:36:43 CST 2025] <<<<<< run test case event_filter success! >>>>>> table processor_delay.t39 exists table processor_delay.t40 not exists for 1-th check, retry later table ddl_manager.finish_mark not exists for 77-th check, retry later table force_replicate_table.t6 exists check_data_subset force_replicate_table.t0 127.0.0.1 4000 127.0.0.1 3306 Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) run task successfully check_data_subset force_replicate_table.t1 127.0.0.1 4000 127.0.0.1 3306 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/ci_collation_compatibility/run.sh using Sink-Type: pulsar... <<================= [Fri Apr 4 15:36:45 CST 2025] <<<<<< run test case ci_collation_compatibility success! >>>>>> table processor_delay.t40 exists table processor_delay.t41 not exists for 1-th check, retry later changefeed count 0 check pass, pd_addr: http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_with_auth.cli.13774.out cli changefeed create '--sink-uri=pulsar://127.0.0.1:6650/ticdc-cli-test-26121?protocol=canal-json&enable-tidb-extension=true' --tz=Asia/Shanghai -c=custom-changefeed-name [WARN] --tz is deprecated in changefeed settings. run task successfully check_data_subset force_replicate_table.t2 127.0.0.1 4000 127.0.0.1 3306 Create changefeed successfully! ID: custom-changefeed-name Info: {"upstream_id":7489358500607251201,"namespace":"default","id":"custom-changefeed-name","sink_uri":"pulsar://127.0.0.1:6650/ticdc-cli-test-26121?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2025-04-04T15:36:46.3243193+08:00","start_ts":457114178356510721,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"pulsar_config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-10-ga306f4894","resolved_ts":457114178356510721,"checkpoint_ts":457114178356510721,"checkpoint_time":"2025-04-04 15:36:46.255"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... table ddl_manager.finish_mark not exists for 78-th check, retry later run task successfully check_data_subset force_replicate_table.t3 127.0.0.1 4000 127.0.0.1 3306 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489358678797910232 TiDB Cluster ID. tikv_gc_leader_uuid 657feec5ecc000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-4ttz-989qr, pid:12210, start at 2025-04-04 15:36:46.27564766 +0800 CST m=+1.578074673 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:38:46.283 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:36:46.259 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:26:46.259 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489358678797910232 TiDB Cluster ID. tikv_gc_leader_uuid 657feec5ecc000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-4ttz-989qr, pid:12210, start at 2025-04-04 15:36:46.27564766 +0800 CST m=+1.578074673 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:38:46.283 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:36:46.259 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:26:46.259 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489358682196451085 TiDB Cluster ID. tikv_gc_leader_uuid 657feec606c0003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-4ttz-989qr, pid:12274, start at 2025-04-04 15:36:46.367402225 +0800 CST m=+1.606239993 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:38:46.378 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:36:46.363 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:26:46.363 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/resourcecontrol/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/resourcecontrol/tiflash/log/error.log arg matches is ArgMatches { args: {"memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/resourcecontrol/tiflash/db/proxy"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/resourcecontrol/tiflash/log/proxy.log"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/resourcecontrol/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table processor_delay.t41 exists table processor_delay.t42 exists table processor_delay.t43 not exists for 1-th check, retry later run task successfully check_data_subset force_replicate_table.t4 127.0.0.1 4000 127.0.0.1 3306 + set +x table ddl_manager.finish_mark exists check diff successfully run task successfully check_data_subset force_replicate_table.t5 127.0.0.1 4000 127.0.0.1 3306 \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-3484/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // timeout [Pipeline] } wait process cdc.test exit for 1-th time... [Pipeline] // stage + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.resourcecontrol.cli.13650.out cli tso query --pd=http://127.0.0.1:2379 [Pipeline] } [Pipeline] // container [Pipeline] } wait process cdc.test exit for 2-th time... [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } table processor_delay.t43 exists table processor_delay.t44 not exists for 1-th check, retry later run task successfully check_data_subset force_replicate_table.t6 127.0.0.1 4000 127.0.0.1 3306 id=7,a=NULL doesn't exist in downstream table force_replicate_table.t6 run task failed 1-th time, retry later [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } wait process cdc.test exit for 3-th time... wait process cdc.test exit for 4-th time... wait process cdc.test exit for 5-th time... + set +x + tso='457114179196420097 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 457114179196420097 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 4 15:36:50 CST 2025] <<<<<< START cdc server in resourcecontrol case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.resourcecontrol.1368713689.out server --log-file /tmp/tidb_cdc_test/resourcecontrol/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/resourcecontrol/cdc_data --cluster-id default + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + endpoints=http://127.0.0.1:2379 + changefeed_id=custom-changefeed-name + expected_state=normal + error_msg=null + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c custom-changefeed-name -s + info='{ "upstream_id": 7489358500607251201, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 457114179247800327, "checkpoint_time": "2025-04-04 15:36:49.655", "error": null }' + echo '{ "upstream_id": 7489358500607251201, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 457114179247800327, "checkpoint_time": "2025-04-04 15:36:49.655", "error": null }' { "upstream_id": 7489358500607251201, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 457114179247800327, "checkpoint_time": "2025-04-04 15:36:49.655", "error": null } ++ echo '{' '"upstream_id":' 7489358500607251201, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 457114179247800327, '"checkpoint_time":' '"2025-04-04' '15:36:49.655",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7489358500607251201, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 457114179247800327, '"checkpoint_time":' '"2025-04-04' '15:36:49.655",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] table processor_delay.t44 exists table processor_delay.t45 not exists for 1-th check, retry later check_data_subset force_replicate_table.t6 127.0.0.1 4000 127.0.0.1 3306 wait process cdc.test exit for 6-th time... + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_with_auth.cli.13861.out cli unsafe delete-service-gc-safepoint =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/move_table/run.sh using Sink-Type: pulsar... <<================= The 1 times to try to start tidb cluster... wait process cdc.test exit for 7-th time... Confirm that you know what this command will do and use it at your own risk [Y/N] CDC service GC safepoint truncated in PD! PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... run task successfully wait process cdc.test exit for 8-th time... wait process cdc.test exit for 1-th time... cdc.test: no process found wait process cdc.test exit for 9-th time... process cdc.test already exit [Fri Apr 4 15:36:53 CST 2025] <<<<<< run test case ddl_manager success! >>>>>> table processor_delay.t45 exists table processor_delay.t46 not exists for 1-th check, retry later wait process cdc.test exit for 2-th time... + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_with_auth.cli.13884.out cli unsafe reset --no-confirm --pd=http://127.0.0.1:2379 cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 4 15:36:53 CST 2025] <<<<<< run test case force_replicate_table success! >>>>>> reset and all metadata truncated in PD! PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 07:36:53 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [817 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7a33522d-7fdc-48f8-8874-1d9c678332c6 {"id":"7a33522d-7fdc-48f8-8874-1d9c678332c6","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752211} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffbafd7fd7 7a33522d-7fdc-48f8-8874-1d9c678332c6 /tidb/cdc/default/default/upstream/7489358678797910232 {"id":7489358678797910232,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7a33522d-7fdc-48f8-8874-1d9c678332c6 {"id":"7a33522d-7fdc-48f8-8874-1d9c678332c6","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752211} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffbafd7fd7 7a33522d-7fdc-48f8-8874-1d9c678332c6 /tidb/cdc/default/default/upstream/7489358678797910232 {"id":7489358678797910232,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7a33522d-7fdc-48f8-8874-1d9c678332c6 {"id":"7a33522d-7fdc-48f8-8874-1d9c678332c6","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752211} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffbafd7fd7 7a33522d-7fdc-48f8-8874-1d9c678332c6 /tidb/cdc/default/default/upstream/7489358678797910232 {"id":7489358678797910232,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x ++ date + echo '[Fri Apr 4 15:36:53 CST 2025] <<<<<< START pulsar cluster in resourcecontrol case >>>>>>' [Fri Apr 4 15:36:53 CST 2025] <<<<<< START pulsar cluster in resourcecontrol case >>>>>> + workdir=/tmp/tidb_cdc_test/resourcecontrol + cluster_type=normal + cd /tmp/tidb_cdc_test/resourcecontrol + DEFAULT_PULSAR_HOME=/usr/local/pulsar + pulsar_dir=/usr/local/pulsar ++ cat + mtls_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderTls brokerClientTlsEnabled=true brokerClientTrustCertsFilePath=/tmp/tidb_cdc_test/resourcecontrol/ca.cert.pem brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls brokerClientAuthenticationParameters={"tlsCertFile":"/tmp/tidb_cdc_test/resourcecontrol/broker_client.cert.pem","tlsKeyFile":"/tmp/tidb_cdc_test/resourcecontrol/broker_client.key-pk8.pem"} brokerServicePortTls=6651 webServicePortTls=8443 tlsTrustCertsFilePath=/tmp/tidb_cdc_test/resourcecontrol/ca.cert.pem tlsCertificateFilePath=/tmp/tidb_cdc_test/resourcecontrol/server.cert.pem tlsKeyFilePath=/tmp/tidb_cdc_test/resourcecontrol/server.key-pk8.pem tlsRequireTrustedClientCertOnConnect=true tlsAllowInsecureConnection=false tlsCertRefreshCheckDurationSec=300' ++ cat + normal_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' ++ cat + mtls_client_conf=' webServiceUrl=https://localhost:8443/ brokerServiceUrl=pulsar+ssl://localhost:6651/ authPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls authParams=tlsCertFile:/tmp/tidb_cdc_test/resourcecontrol/broker_client.cert.pem,tlsKeyFile:/tmp/tidb_cdc_test/resourcecontrol/broker_client.key-pk8.pem tlsTrustCertsFilePath=/tmp/tidb_cdc_test/resourcecontrol/ca.cert.pem' ++ cat + oauth_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/ authPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 authParams={"privateKey":"/tmp/tidb_cdc_test/resourcecontrol/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"}' ++ cat + oauth_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderToken brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 brokerClientAuthenticationParameters={"privateKey":"file:///tmp/tidb_cdc_test/resourcecontrol/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"} tokenSecretKey=data:;base64,U0poWDM2X0thcFlTeWJCdEpxMzVseFhfQnJyNExSVVJTa203UW1YSkdteThwVUZXOUVJT2NWUVBzeWt6OS1qag==' ++ cat + credential_json=' { "client_id":"1234", "client_secret":"e0KVlA2EiBfjoN13olyZd2kv1KL", "audience":"cdc-api-uri", "issuer_url":"http://localhost:9096", "type": "client_credentials" }' ++ cat + cert_server_conf='[ req ] default_bits = 2048 prompt = no default_md = sha256 distinguished_name = dn [ v3_ext ] authorityKeyIdentifier=keyid,issuer:always basicConstraints=CA:FALSE keyUsage=critical, digitalSignature, keyEncipherment extendedKeyUsage=serverAuth subjectAltName=@alt_names [ dn ] CN = server [ alt_names ] DNS.1 = localhost IP.1 = 127.0.0.1' + echo ' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' + cp /usr/local/pulsar/conf/standalone.conf /tmp/tidb_cdc_test/resourcecontrol/pulsar_standalone.conf + pulsar_port=6650 + '[' normal == mtls ']' + '[' normal == oauth ']' + echo 'no cluster type specified, using default configuration.' no cluster type specified, using default configuration. ++ date + echo '[Fri Apr 4 15:36:54 CST 2025] <<<<<< START pulsar cluster in normal mode in resourcecontrol case >>>>>>' [Fri Apr 4 15:36:54 CST 2025] <<<<<< START pulsar cluster in normal mode in resourcecontrol case >>>>>> + echo 'Waiting for pulsar port to be ready...' Waiting for pulsar port to be ready... + i=0 + nc -z localhost 6650 + /usr/local/pulsar/bin/pulsar standalone --config /tmp/tidb_cdc_test/resourcecontrol/pulsar_standalone.conf -nfw --metadata-dir /tmp/tidb_cdc_test/resourcecontrol/pulsar-metadata --bookkeeper-dir /tmp/tidb_cdc_test/resourcecontrol/pulsar-bookie + i=1 + '[' 1 -gt 20 ']' + sleep 2 start tidb cluster in /tmp/tidb_cdc_test/move_table Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/sql_mode/run.sh using Sink-Type: pulsar... <<================= +++ dirname /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/sql_mode/run.sh ++ cd /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/sql_mode ++ pwd + CUR=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/sql_mode + source /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/sql_mode/../_utils/test_prepare ++ UP_TIDB_HOST=127.0.0.1 ++ UP_TIDB_PORT=4000 ++ UP_TIDB_OTHER_PORT=4001 ++ UP_TIDB_STATUS=10080 ++ UP_TIDB_OTHER_STATUS=10081 ++ DOWN_TIDB_HOST=127.0.0.1 ++ DOWN_TIDB_PORT=3306 ++ DOWN_TIDB_STATUS=20080 ++ TLS_TIDB_HOST=127.0.0.1 ++ TLS_TIDB_PORT=3307 ++ TLS_TIDB_STATUS=30080 ++ UP_PD_HOST_1=127.0.0.1 ++ UP_PD_PORT_1=2379 ++ UP_PD_PEER_PORT_1=2380 ++ UP_PD_HOST_2=127.0.0.1 ++ UP_PD_PORT_2=2679 ++ UP_PD_PEER_PORT_2=2680 ++ UP_PD_HOST_3=127.0.0.1 ++ UP_PD_PORT_3=2779 ++ UP_PD_PEER_PORT_3=2780 ++ DOWN_PD_HOST=127.0.0.1 ++ DOWN_PD_PORT=2479 ++ DOWN_PD_PEER_PORT=2480 ++ TLS_PD_HOST=127.0.0.1 ++ TLS_PD_PORT=2579 ++ TLS_PD_PEER_PORT=2580 ++ UP_TIKV_HOST_1=127.0.0.1 ++ UP_TIKV_PORT_1=20160 ++ UP_TIKV_STATUS_PORT_1=20181 ++ UP_TIKV_HOST_2=127.0.0.1 ++ UP_TIKV_PORT_2=20161 ++ UP_TIKV_STATUS_PORT_2=20182 ++ UP_TIKV_HOST_3=127.0.0.1 ++ UP_TIKV_PORT_3=20162 ++ UP_TIKV_STATUS_PORT_3=20183 ++ DOWN_TIKV_HOST=127.0.0.1 ++ DOWN_TIKV_PORT=21160 ++ DOWN_TIKV_STATUS_PORT=21180 ++ TLS_TIKV_HOST=127.0.0.1 ++ TLS_TIKV_PORT=22160 ++ TLS_TIKV_STATUS_PORT=22180 +++ cat /tmp/tidb_cdc_test/KAFKA_VERSION +++ echo 2.4.1 ++ KAFKA_VERSION=2.4.1 + WORK_DIR=/tmp/tidb_cdc_test/sql_mode + CDC_BINARY=cdc.test + SINK_TYPE=pulsar + CDC_COUNT=3 + DB_COUNT=4 + rm -rf /tmp/tidb_cdc_test/sql_mode + mkdir -p /tmp/tidb_cdc_test/sql_mode + start_tidb_cluster --workdir /tmp/tidb_cdc_test/sql_mode The 1 times to try to start tidb cluster... + set +x table processor_delay.t46 exists table processor_delay.t47 not exists for 1-th check, retry later + nc -z localhost 6650 + i=2 + '[' 2 -gt 20 ']' + sleep 2 table processor_delay.t47 exists table processor_delay.t48 not exists for 1-th check, retry later + nc -z localhost 6650 + echo 'Waiting for pulsar namespace to be ready...' Waiting for pulsar namespace to be ready... + i=0 + /usr/local/pulsar/bin/pulsar-admin namespaces list public TEST FAILED: OUTPUT DOES NOT CONTAIN 'id: 1' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ check data failed 1-th time, retry later check data successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 4 15:36:55 CST 2025] <<<<<< run test case ddl_puller_lag success! >>>>>> start tidb cluster in /tmp/tidb_cdc_test/sql_mode Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_with_auth.cli.13973.out cli unsafe resolve-lock --region=38 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... table processor_delay.t48 exists table processor_delay.t49 not exists for 1-th check, retry later + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_with_auth.cli.14008.out cli unsafe resolve-lock --region=38 --ts=457114180729176073 public/default ++ date + echo '[Fri Apr 4 15:37:00 CST 2025] <<<<<< pulsar is ready >>>>>>' [Fri Apr 4 15:37:00 CST 2025] <<<<<< pulsar is ready >>>>>> + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.resourcecontrol.cli.14153.out cli changefeed create --start-ts=457114179196420097 '--sink-uri=pulsar://127.0.0.1:6650/ticdc-resourcecontrol-test-24198?protocol=canal-json&enable-tidb-extension=true' PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table processor_delay.t49 exists table processor_delay.t50 not exists for 1-th check, retry later Create changefeed successfully! ID: 9e0779b9-5da9-4bab-b152-53145645ee9f Info: {"upstream_id":7489358678797910232,"namespace":"default","id":"9e0779b9-5da9-4bab-b152-53145645ee9f","sink_uri":"pulsar://127.0.0.1:6650/ticdc-resourcecontrol-test-24198?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2025-04-04T15:37:01.422577732+08:00","start_ts":457114179196420097,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"pulsar_config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-10-ga306f4894","resolved_ts":457114179196420097,"checkpoint_ts":457114179196420097,"checkpoint_time":"2025-04-04 15:36:49.459"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... + set +x % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 218 100 212 100 6 207k 6000 --:--:-- --:--:-- --:--:-- 212k { "error_msg": "[CDC:ErrAPIInvalidParam]invalid log level: json: cannot unmarshal string into Go value of type struct { Level string \"json:\\\"log_level\\\"\" }", "error_code": "CDC:ErrAPIInvalidParam" Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x [Fri Apr 4 15:37:02 CST 2025] <<<<<< START Pulsar consumer in resourcecontrol case >>>>>> =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/multi_source/run.sh using Sink-Type: pulsar... <<================= table processor_delay.t50 exists check diff failed 1-th time, retry later Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) } % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 183 100 183 0 0 178k 0 --:--:-- --:--:-- --:--:-- 178k { "version": "v9.0.0-beta.1-10-ga306f4894", "git_hash": "a306f48949e037965a7c1963f500f7e0abbfba51", "id": "5586c95f-7ddc-4c28-a921-4a04ad6b2486", "pid": 12743, "is_owner": true table resourcecontrol.finish_mark not exists for 1-th check, retry later }wait process cdc.test exit for 1-th time... \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-3484/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } wait process cdc.test exit for 2-th time... [Pipeline] // dir [Pipeline] } =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/vector/run.sh using Sink-Type: pulsar... <<================= The 1 times to try to start tidb cluster... [Pipeline] // timeout =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/ddl_only_block_related_table/run.sh using Sink-Type: pulsar... <<================= The 1 times to try to start tidb cluster... [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489358765073995196 TiDB Cluster ID. tikv_gc_leader_uuid 657feed9390000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-b84s-w4c6p, pid:10971, start at 2025-04-04 15:37:06.064460179 +0800 CST m=+1.567529247 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:39:06.072 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:37:06.070 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:27:06.070 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489358765073995196 TiDB Cluster ID. tikv_gc_leader_uuid 657feed9390000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-b84s-w4c6p, pid:10971, start at 2025-04-04 15:37:06.064460179 +0800 CST m=+1.567529247 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:39:06.072 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:37:06.070 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:27:06.070 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489358753973086584 TiDB Cluster ID. tikv_gc_leader_uuid 657feed94540003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-b84s-w4c6p, pid:11047, start at 2025-04-04 15:37:06.072486896 +0800 CST m=+1.500698448 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:39:06.083 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:37:06.069 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:27:06.069 +0800 All versions after safe point can be accessed. (DO NOT EDIT) [Pipeline] // node ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] } cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 4 15:37:06 CST 2025] <<<<<< run test case cli_with_auth success! >>>>>> [Pipeline] // podTemplate [Pipeline] } Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/sql_mode/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/sql_mode/tiflash/log/error.log arg matches is ArgMatches { args: {"memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/sql_mode/tiflash/log/proxy.log"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/sql_mode/tiflash-proxy.toml"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/sql_mode/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } check diff successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... table resourcecontrol.finish_mark exists check diff successfully wait process cdc.test exit for 1-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 4 15:37:08 CST 2025] <<<<<< run test case processor_etcd_worker_delay success! >>>>>> wait process cdc.test exit for 2-th time... start tidb cluster in /tmp/tidb_cdc_test/ddl_only_block_related_table Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 4 15:37:09 CST 2025] <<<<<< run test case resourcecontrol success! >>>>>> + trap stop_tidb_cluster EXIT + run_sql 'set global sql_mode='\''NO_BACKSLASH_ESCAPES'\'';' 127.0.0.1 4000 + run_sql 'set global sql_mode='\''NO_BACKSLASH_ESCAPES'\'';' 127.0.0.1 3306 + cd /tmp/tidb_cdc_test/sql_mode ++ run_cdc_cli_tso_query 127.0.0.1 2379 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sql_mode.cli.12435.out cli tso query --pd=http://127.0.0.1:2379 start tidb cluster in /tmp/tidb_cdc_test/vector Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... + set +x + tso='457114184390017025 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 457114184390017025 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + start_ts=457114184390017025 + run_cdc_server --workdir /tmp/tidb_cdc_test/sql_mode --binary cdc.test [Fri Apr 4 15:37:10 CST 2025] <<<<<< START cdc server in sql_mode case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sql_mode.1247212475.out server --log-file /tmp/tidb_cdc_test/sql_mode/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/sql_mode/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL Verifying downstream PD is started... * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 07:37:13 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [817 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/43dfc674-a146-4638-9ad3-f34f3a58ef08 {"id":"43dfc674-a146-4638-9ad3-f34f3a58ef08","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752231} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffbb46fdd9 43dfc674-a146-4638-9ad3-f34f3a58ef08 /tidb/cdc/default/default/upstream/7489358765073995196 {"id":7489358765073995196,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/43dfc674-a146-4638-9ad3-f34f3a58ef08 {"id":"43dfc674-a146-4638-9ad3-f34f3a58ef08","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752231} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffbb46fdd9 43dfc674-a146-4638-9ad3-f34f3a58ef08 /tidb/cdc/default/default/upstream/7489358765073995196 {"id":7489358765073995196,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/43dfc674-a146-4638-9ad3-f34f3a58ef08 {"id":"43dfc674-a146-4638-9ad3-f34f3a58ef08","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752231} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffbb46fdd9 43dfc674-a146-4638-9ad3-f34f3a58ef08 /tidb/cdc/default/default/upstream/7489358765073995196 {"id":7489358765073995196,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + SINK_URI='mysql://root@127.0.0.1:3306/?max-txn-row=1' + run_cdc_cli changefeed create --start-ts=457114184390017025 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sql_mode.cli.12531.out cli changefeed create --start-ts=457114184390017025 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 Create changefeed successfully! ID: test-1 Info: {"upstream_id":7489358765073995196,"namespace":"default","id":"test-1","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2025-04-04T15:37:14.304251527+08:00","start_ts":457114184390017025,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-10-ga306f4894","resolved_ts":457114184390017025,"checkpoint_ts":457114184390017025,"checkpoint_time":"2025-04-04 15:37:09.271"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x + run_sql 'use test; create table t1(id bigint primary key, a text, b text as ((regexp_replace(a, '\''^[1-9]\d{9,29}$'\'', '\''aaaaa'\''))), c text); insert into t1 (id, a, c) values(1,123456, '\''ab\\\\c'\''); insert into t1 (id, a, c) values(2,1234567890123, '\''ab\\c'\'');' 127.0.0.1 4000 + '[' pulsar == mysql ']' + stop_tidb_cluster =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/savepoint/run.sh using Sink-Type: pulsar... <<================= The 1 times to try to start tidb cluster... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489358796754131989 TiDB Cluster ID. tikv_gc_leader_uuid 657feee3024000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-sfgg-lkh0k, pid:8342, start at 2025-04-04 15:37:16.058995207 +0800 CST m=+1.572987541 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:39:16.066 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:37:16.041 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:27:16.041 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489358796754131989 TiDB Cluster ID. tikv_gc_leader_uuid 657feee3024000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-sfgg-lkh0k, pid:8342, start at 2025-04-04 15:37:16.058995207 +0800 CST m=+1.572987541 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:39:16.066 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:37:16.041 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:27:16.041 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489358806281329385 TiDB Cluster ID. tikv_gc_leader_uuid 657feee30d00009 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-sfgg-lkh0k, pid:8415, start at 2025-04-04 15:37:16.098995164 +0800 CST m=+1.531642163 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:39:16.108 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:37:16.084 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:27:16.084 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/vector/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/vector/tiflash/log/error.log arg matches is ArgMatches { args: {"log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/vector/tiflash/log/proxy.log"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/vector/tiflash/db/proxy"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/vector/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489358799122364535 TiDB Cluster ID. tikv_gc_leader_uuid 657feee35fc000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-2bnf-6v9zm, pid:7990, start at 2025-04-04 15:37:16.43900136 +0800 CST m=+1.444814350 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:39:16.446 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:37:16.415 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:27:16.415 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489358799122364535 TiDB Cluster ID. tikv_gc_leader_uuid 657feee35fc000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-2bnf-6v9zm, pid:7990, start at 2025-04-04 15:37:16.43900136 +0800 CST m=+1.444814350 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:39:16.446 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:37:16.415 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:27:16.415 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489358807691159771 TiDB Cluster ID. tikv_gc_leader_uuid 657feee3854000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-2bnf-6v9zm, pid:8076, start at 2025-04-04 15:37:16.593752164 +0800 CST m=+1.524482294 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:39:16.602 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:37:16.565 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:27:16.565 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/ddl_only_block_related_table/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/ddl_only_block_related_table/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/ddl_only_block_related_table/tiflash/log/proxy.log"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/ddl_only_block_related_table/tiflash-proxy.toml"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/ddl_only_block_related_table/tiflash/db/proxy"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.vector.cli.9869.out cli tso query --pd=http://127.0.0.1:2379 \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-3484/tiflow-cdc already exists) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Fri Apr 4 15:37:19 CST 2025] <<<<<< START cdc server in ddl_only_block_related_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_only_block_related_table.95359537.out server --log-file /tmp/tidb_cdc_test/ddl_only_block_related_table/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_only_block_related_table/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv start tidb cluster in /tmp/tidb_cdc_test/savepoint Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 [Pipeline] } [Pipeline] // stage [Pipeline] } Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/sink_hang/run.sh using Sink-Type: pulsar... <<================= The 1 times to try to start tidb cluster... + set +x + tso='457114187003592705 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 457114187003592705 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 4 15:37:20 CST 2025] <<<<<< START cdc server in vector case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.vector.99049906.out server --log-file /tmp/tidb_cdc_test/vector/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/vector/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 07:37:22 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [817 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/aecc9afe-1586-4b0d-909e-d8dabce4d80c {"id":"aecc9afe-1586-4b0d-909e-d8dabce4d80c","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752239} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffbb6fc9ce aecc9afe-1586-4b0d-909e-d8dabce4d80c /tidb/cdc/default/default/upstream/7489358799122364535 {"id":7489358799122364535,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/aecc9afe-1586-4b0d-909e-d8dabce4d80c {"id":"aecc9afe-1586-4b0d-909e-d8dabce4d80c","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752239} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffbb6fc9ce aecc9afe-1586-4b0d-909e-d8dabce4d80c /tidb/cdc/default/default/upstream/7489358799122364535 {"id":7489358799122364535,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/aecc9afe-1586-4b0d-909e-d8dabce4d80c {"id":"aecc9afe-1586-4b0d-909e-d8dabce4d80c","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752239} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffbb6fc9ce aecc9afe-1586-4b0d-909e-d8dabce4d80c /tidb/cdc/default/default/upstream/7489358799122364535 {"id":7489358799122364535,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x ++ date + echo '[Fri Apr 4 15:37:22 CST 2025] <<<<<< START pulsar cluster in ddl_only_block_related_table case >>>>>>' [Fri Apr 4 15:37:22 CST 2025] <<<<<< START pulsar cluster in ddl_only_block_related_table case >>>>>> + workdir=/tmp/tidb_cdc_test/ddl_only_block_related_table + cluster_type=normal + cd /tmp/tidb_cdc_test/ddl_only_block_related_table + DEFAULT_PULSAR_HOME=/usr/local/pulsar + pulsar_dir=/usr/local/pulsar ++ cat + mtls_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderTls brokerClientTlsEnabled=true brokerClientTrustCertsFilePath=/tmp/tidb_cdc_test/ddl_only_block_related_table/ca.cert.pem brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls brokerClientAuthenticationParameters={"tlsCertFile":"/tmp/tidb_cdc_test/ddl_only_block_related_table/broker_client.cert.pem","tlsKeyFile":"/tmp/tidb_cdc_test/ddl_only_block_related_table/broker_client.key-pk8.pem"} brokerServicePortTls=6651 webServicePortTls=8443 tlsTrustCertsFilePath=/tmp/tidb_cdc_test/ddl_only_block_related_table/ca.cert.pem tlsCertificateFilePath=/tmp/tidb_cdc_test/ddl_only_block_related_table/server.cert.pem tlsKeyFilePath=/tmp/tidb_cdc_test/ddl_only_block_related_table/server.key-pk8.pem tlsRequireTrustedClientCertOnConnect=true tlsAllowInsecureConnection=false tlsCertRefreshCheckDurationSec=300' ++ cat + normal_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' ++ cat + mtls_client_conf=' webServiceUrl=https://localhost:8443/ brokerServiceUrl=pulsar+ssl://localhost:6651/ authPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls authParams=tlsCertFile:/tmp/tidb_cdc_test/ddl_only_block_related_table/broker_client.cert.pem,tlsKeyFile:/tmp/tidb_cdc_test/ddl_only_block_related_table/broker_client.key-pk8.pem tlsTrustCertsFilePath=/tmp/tidb_cdc_test/ddl_only_block_related_table/ca.cert.pem' ++ cat + oauth_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/ authPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 authParams={"privateKey":"/tmp/tidb_cdc_test/ddl_only_block_related_table/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"}' ++ cat + oauth_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderToken brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 brokerClientAuthenticationParameters={"privateKey":"file:///tmp/tidb_cdc_test/ddl_only_block_related_table/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"} tokenSecretKey=data:;base64,U0poWDM2X0thcFlTeWJCdEpxMzVseFhfQnJyNExSVVJTa203UW1YSkdteThwVUZXOUVJT2NWUVBzeWt6OS1qag==' ++ cat + credential_json=' { "client_id":"1234", "client_secret":"e0KVlA2EiBfjoN13olyZd2kv1KL", "audience":"cdc-api-uri", "issuer_url":"http://localhost:9096", "type": "client_credentials" }' ++ cat + cert_server_conf='[ req ] default_bits = 2048 prompt = no default_md = sha256 distinguished_name = dn [ v3_ext ] authorityKeyIdentifier=keyid,issuer:always basicConstraints=CA:FALSE keyUsage=critical, digitalSignature, keyEncipherment extendedKeyUsage=serverAuth subjectAltName=@alt_names [ dn ] CN = server [ alt_names ] DNS.1 = localhost IP.1 = 127.0.0.1' + echo ' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' + cp /usr/local/pulsar/conf/standalone.conf /tmp/tidb_cdc_test/ddl_only_block_related_table/pulsar_standalone.conf + pulsar_port=6650 + '[' normal == mtls ']' + '[' normal == oauth ']' + echo 'no cluster type specified, using default configuration.' no cluster type specified, using default configuration. ++ date + echo '[Fri Apr 4 15:37:22 CST 2025] <<<<<< START pulsar cluster in normal mode in ddl_only_block_related_table case >>>>>>' [Fri Apr 4 15:37:22 CST 2025] <<<<<< START pulsar cluster in normal mode in ddl_only_block_related_table case >>>>>> + echo 'Waiting for pulsar port to be ready...' Waiting for pulsar port to be ready... + i=0 + nc -z localhost 6650 + /usr/local/pulsar/bin/pulsar standalone --config /tmp/tidb_cdc_test/ddl_only_block_related_table/pulsar_standalone.conf -nfw --metadata-dir /tmp/tidb_cdc_test/ddl_only_block_related_table/pulsar-metadata --bookkeeper-dir /tmp/tidb_cdc_test/ddl_only_block_related_table/pulsar-bookie + i=1 + '[' 1 -gt 20 ']' + sleep 2 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/sink_hang Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 07:37:23 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [817 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/08112eb1-d1e0-4947-a106-ac61e2fc06a7 {"id":"08112eb1-d1e0-4947-a106-ac61e2fc06a7","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752241} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffbb71d5da 08112eb1-d1e0-4947-a106-ac61e2fc06a7 /tidb/cdc/default/default/upstream/7489358796754131989 {"id":7489358796754131989,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/08112eb1-d1e0-4947-a106-ac61e2fc06a7 {"id":"08112eb1-d1e0-4947-a106-ac61e2fc06a7","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752241} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffbb71d5da 08112eb1-d1e0-4947-a106-ac61e2fc06a7 /tidb/cdc/default/default/upstream/7489358796754131989 {"id":7489358796754131989,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/08112eb1-d1e0-4947-a106-ac61e2fc06a7 {"id":"08112eb1-d1e0-4947-a106-ac61e2fc06a7","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752241} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffbb71d5da 08112eb1-d1e0-4947-a106-ac61e2fc06a7 /tidb/cdc/default/default/upstream/7489358796754131989 {"id":7489358796754131989,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.vector.cli.9965.out cli changefeed create --start-ts=457114187003592705 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ Create changefeed successfully! ID: 7ca24a25-38ce-412d-a243-0ad4901b61c7 Info: {"upstream_id":7489358796754131989,"namespace":"default","id":"7ca24a25-38ce-412d-a243-0ad4901b61c7","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-04T15:37:24.152805335+08:00","start_ts":457114187003592705,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-10-ga306f4894","resolved_ts":457114187003592705,"checkpoint_ts":457114187003592705,"checkpoint_time":"2025-04-04 15:37:19.241"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + nc -z localhost 6650 + i=2 + '[' 2 -gt 20 ']' + sleep 2 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489358746991142230 TiDB Cluster ID. tikv_gc_leader_uuid 657feeeb768000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-6nlr-p88d4, pid:24664, start at 2025-04-04 15:37:24.715075582 +0800 CST m=+24.146373606 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:39:24.726 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:37:24.698 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:27:24.698 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489358746991142230 TiDB Cluster ID. tikv_gc_leader_uuid 657feeeb768000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-6nlr-p88d4, pid:24664, start at 2025-04-04 15:37:24.715075582 +0800 CST m=+24.146373606 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:39:24.726 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:37:24.698 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:27:24.698 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489358745566493425 TiDB Cluster ID. tikv_gc_leader_uuid 657feed5708000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-6nlr-p88d4, pid:24731, start at 2025-04-04 15:37:02.188254433 +0800 CST m=+1.550865600 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:39:02.196 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:37:02.196 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:27:02.196 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/move_table/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/move_table/tiflash/log/error.log arg matches is ArgMatches { args: {"config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/move_table/tiflash-proxy.toml"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/move_table/tiflash/log/proxy.log"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/move_table/tiflash/db/proxy"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + nc -z localhost 6650 + echo 'Waiting for pulsar namespace to be ready...' Waiting for pulsar namespace to be ready... + i=0 + /usr/local/pulsar/bin/pulsar-admin namespaces list public VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489358853626312113 TiDB Cluster ID. tikv_gc_leader_uuid 657feeee778000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-qg8g-pl0pp, pid:15118, start at 2025-04-04 15:37:27.813419942 +0800 CST m=+1.608816515 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:39:27.821 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:37:27.825 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:27:27.825 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489358853626312113 TiDB Cluster ID. tikv_gc_leader_uuid 657feeee778000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-qg8g-pl0pp, pid:15118, start at 2025-04-04 15:37:27.813419942 +0800 CST m=+1.608816515 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:39:27.821 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:37:27.825 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:27:27.825 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489358857653137407 TiDB Cluster ID. tikv_gc_leader_uuid 657feeee78c000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-qg8g-pl0pp, pid:15197, start at 2025-04-04 15:37:27.799000425 +0800 CST m=+1.507952238 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:39:27.806 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:37:27.779 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:27:27.779 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/savepoint/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/savepoint/tiflash/log/error.log arg matches is ArgMatches { args: {"config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/savepoint/tiflash-proxy.toml"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/savepoint/tiflash/db/proxy"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/savepoint/tiflash/log/proxy.log"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table test.finish_mark not exists for 1-th check, retry later Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) public/default ++ date + echo '[Fri Apr 4 15:37:28 CST 2025] <<<<<< pulsar is ready >>>>>>' [Fri Apr 4 15:37:28 CST 2025] <<<<<< pulsar is ready >>>>>> + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_only_block_related_table.cli.10007.out cli changefeed create '--sink-uri=pulsar://127.0.0.1:6650/ticdc-common-1-test-22617?protocol=canal-json&enable-tidb-extension=true' -c=ddl-only-block-related-table + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.move_table.cli.26162.out cli tso query --pd=http://127.0.0.1:2379 Create changefeed successfully! ID: ddl-only-block-related-table Info: {"upstream_id":7489358799122364535,"namespace":"default","id":"ddl-only-block-related-table","sink_uri":"pulsar://127.0.0.1:6650/ticdc-common-1-test-22617?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2025-04-04T15:37:29.729663533+08:00","start_ts":457114189605371907,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"pulsar_config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-10-ga306f4894","resolved_ts":457114189605371907,"checkpoint_ts":457114189605371907,"checkpoint_time":"2025-04-04 15:37:29.166"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... + start_tidb_cluster --workdir /tmp/tidb_cdc_test/sql_mode The 1 times to try to start tidb cluster... + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.savepoint.cli.16575.out cli tso query --pd=http://127.0.0.1:2379 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489358867304288092 TiDB Cluster ID. tikv_gc_leader_uuid 657feef10a8000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-7fxb-ntnhx, pid:8981, start at 2025-04-04 15:37:30.456492022 +0800 CST m=+1.764380833 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:39:30.465 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:37:30.460 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:27:30.460 +0800 All versions after safe point can be accessed. (DO NOT EDIT) + set +x + tso='457114189666189313 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 457114189666189313 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x ***************** properties ***************** "mysql.user"="root" "recordcount"="10000" "dotransactions"="false" "mysql.port"="4000" "threadcount"="10" "readproportion"="0" "updateproportion"="0" "readallfields"="true" "scanproportion"="0" "mysql.db"="move_table" "workload"="core" "insertproportion"="0" "operationcount"="0" "mysql.host"="127.0.0.1" "requestdistribution"="uniform" ********************************************** VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489358867304288092 TiDB Cluster ID. tikv_gc_leader_uuid 657feef10a8000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-7fxb-ntnhx, pid:8981, start at 2025-04-04 15:37:30.456492022 +0800 CST m=+1.764380833 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:39:30.465 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:37:30.460 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:27:30.460 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489358866741188639 TiDB Cluster ID. tikv_gc_leader_uuid 657feef12500006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-7fxb-ntnhx, pid:9055, start at 2025-04-04 15:37:30.524227375 +0800 CST m=+1.747856606 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:39:30.532 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:37:30.516 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:27:30.516 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored + set +x [Fri Apr 4 15:37:31 CST 2025] <<<<<< START Pulsar consumer in ddl_only_block_related_table case >>>>>> Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/sink_hang/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/sink_hang/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/sink_hang/tiflash/db/proxy"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/sink_hang/tiflash/log/proxy.log"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/sink_hang/tiflash-proxy.toml"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table test.finish_mark not exists for 2-th check, retry later table ddl_only_block_related_table.finish_mark not exists for 1-th check, retry later Run finished, takes 1.335660065s INSERT - Takes(s): 1.3, Count: 10000, OPS: 7514.6, Avg(us): 1290, Min(us): 865, Max(us): 5142, 95th(us): 2000, 99th(us): 2000 [Fri Apr 4 15:37:32 CST 2025] <<<<<< START cdc server in move_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.move_table.2623626238.out server --log-file /tmp/tidb_cdc_test/move_table/cdc1.log --log-level debug --data-dir /tmp/tidb_cdc_test/move_table/cdc_data1 --cluster-id default --addr 127.0.0.1:8300 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + set +x + tso='457114190079590401 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 457114190079590401 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 4 15:37:32 CST 2025] <<<<<< START cdc server in savepoint case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.savepoint.1660716609.out server --log-file /tmp/tidb_cdc_test/savepoint/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/savepoint/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [Fri Apr 4 15:37:33 CST 2025] <<<<<< START cdc server in sink_hang case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sink/dmlsink/txn/mysql/MySQLSinkExecDMLError=2*return(true)' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sink_hang.1050710509.out server --log-file /tmp/tidb_cdc_test/sink_hang/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/sink_hang/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 start tidb cluster in /tmp/tidb_cdc_test/sql_mode Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... table test.finish_mark exists check diff successfully table ddl_only_block_related_table.finish_mark not exists for 2-th check, retry later wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 4 15:37:34 CST 2025] <<<<<< run test case vector success! >>>>>> + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 07:37:35 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [817 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/abe8d4f2-261b-4c68-80b4-55d1683d6a7a {"id":"abe8d4f2-261b-4c68-80b4-55d1683d6a7a","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752252} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffbb399f5f abe8d4f2-261b-4c68-80b4-55d1683d6a7a /tidb/cdc/default/default/upstream/7489358746991142230 {"id":7489358746991142230,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/abe8d4f2-261b-4c68-80b4-55d1683d6a7a {"id":"abe8d4f2-261b-4c68-80b4-55d1683d6a7a","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752252} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffbb399f5f abe8d4f2-261b-4c68-80b4-55d1683d6a7a /tidb/cdc/default/default/upstream/7489358746991142230 {"id":7489358746991142230,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/abe8d4f2-261b-4c68-80b4-55d1683d6a7a {"id":"abe8d4f2-261b-4c68-80b4-55d1683d6a7a","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752252} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffbb399f5f abe8d4f2-261b-4c68-80b4-55d1683d6a7a /tidb/cdc/default/default/upstream/7489358746991142230 {"id":7489358746991142230,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x ++ date + echo '[Fri Apr 4 15:37:35 CST 2025] <<<<<< START pulsar cluster in move_table case >>>>>>' [Fri Apr 4 15:37:35 CST 2025] <<<<<< START pulsar cluster in move_table case >>>>>> + workdir=/tmp/tidb_cdc_test/move_table + cluster_type=normal + cd /tmp/tidb_cdc_test/move_table + DEFAULT_PULSAR_HOME=/usr/local/pulsar + pulsar_dir=/usr/local/pulsar ++ cat + mtls_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderTls brokerClientTlsEnabled=true brokerClientTrustCertsFilePath=/tmp/tidb_cdc_test/move_table/ca.cert.pem brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls brokerClientAuthenticationParameters={"tlsCertFile":"/tmp/tidb_cdc_test/move_table/broker_client.cert.pem","tlsKeyFile":"/tmp/tidb_cdc_test/move_table/broker_client.key-pk8.pem"} brokerServicePortTls=6651 webServicePortTls=8443 tlsTrustCertsFilePath=/tmp/tidb_cdc_test/move_table/ca.cert.pem tlsCertificateFilePath=/tmp/tidb_cdc_test/move_table/server.cert.pem tlsKeyFilePath=/tmp/tidb_cdc_test/move_table/server.key-pk8.pem tlsRequireTrustedClientCertOnConnect=true tlsAllowInsecureConnection=false tlsCertRefreshCheckDurationSec=300' ++ cat + normal_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' ++ cat + mtls_client_conf=' webServiceUrl=https://localhost:8443/ brokerServiceUrl=pulsar+ssl://localhost:6651/ authPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls authParams=tlsCertFile:/tmp/tidb_cdc_test/move_table/broker_client.cert.pem,tlsKeyFile:/tmp/tidb_cdc_test/move_table/broker_client.key-pk8.pem tlsTrustCertsFilePath=/tmp/tidb_cdc_test/move_table/ca.cert.pem' ++ cat + oauth_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/ authPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 authParams={"privateKey":"/tmp/tidb_cdc_test/move_table/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"}' ++ cat + oauth_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderToken brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 brokerClientAuthenticationParameters={"privateKey":"file:///tmp/tidb_cdc_test/move_table/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"} tokenSecretKey=data:;base64,U0poWDM2X0thcFlTeWJCdEpxMzVseFhfQnJyNExSVVJTa203UW1YSkdteThwVUZXOUVJT2NWUVBzeWt6OS1qag==' ++ cat + credential_json=' { "client_id":"1234", "client_secret":"e0KVlA2EiBfjoN13olyZd2kv1KL", "audience":"cdc-api-uri", "issuer_url":"http://localhost:9096", "type": "client_credentials" }' ++ cat + cert_server_conf='[ req ] default_bits = 2048 prompt = no default_md = sha256 distinguished_name = dn [ v3_ext ] authorityKeyIdentifier=keyid,issuer:always basicConstraints=CA:FALSE keyUsage=critical, digitalSignature, keyEncipherment extendedKeyUsage=serverAuth subjectAltName=@alt_names [ dn ] CN = server [ alt_names ] DNS.1 = localhost IP.1 = 127.0.0.1' + echo ' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' + cp /usr/local/pulsar/conf/standalone.conf /tmp/tidb_cdc_test/move_table/pulsar_standalone.conf + pulsar_port=6650 + '[' normal == mtls ']' + '[' normal == oauth ']' + echo 'no cluster type specified, using default configuration.' no cluster type specified, using default configuration. ++ date + echo '[Fri Apr 4 15:37:35 CST 2025] <<<<<< START pulsar cluster in normal mode in move_table case >>>>>>' [Fri Apr 4 15:37:35 CST 2025] <<<<<< START pulsar cluster in normal mode in move_table case >>>>>> + echo 'Waiting for pulsar port to be ready...' Waiting for pulsar port to be ready... + i=0 + nc -z localhost 6650 + /usr/local/pulsar/bin/pulsar standalone --config /tmp/tidb_cdc_test/move_table/pulsar_standalone.conf -nfw --metadata-dir /tmp/tidb_cdc_test/move_table/pulsar-metadata --bookkeeper-dir /tmp/tidb_cdc_test/move_table/pulsar-bookie + i=1 + '[' 1 -gt 20 ']' + sleep 2 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 07:37:35 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [817 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/18964cfc-ec8a-432b-aee7-4152e004cb87 {"id":"18964cfc-ec8a-432b-aee7-4152e004cb87","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752252} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffbb9b93d9 18964cfc-ec8a-432b-aee7-4152e004cb87 /tidb/cdc/default/default/upstream/7489358853626312113 {"id":7489358853626312113,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/18964cfc-ec8a-432b-aee7-4152e004cb87 {"id":"18964cfc-ec8a-432b-aee7-4152e004cb87","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752252} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffbb9b93d9 18964cfc-ec8a-432b-aee7-4152e004cb87 /tidb/cdc/default/default/upstream/7489358853626312113 {"id":7489358853626312113,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/18964cfc-ec8a-432b-aee7-4152e004cb87 {"id":"18964cfc-ec8a-432b-aee7-4152e004cb87","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752252} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffbb9b93d9 18964cfc-ec8a-432b-aee7-4152e004cb87 /tidb/cdc/default/default/upstream/7489358853626312113 {"id":7489358853626312113,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x ++ date + echo '[Fri Apr 4 15:37:35 CST 2025] <<<<<< START pulsar cluster in savepoint case >>>>>>' [Fri Apr 4 15:37:35 CST 2025] <<<<<< START pulsar cluster in savepoint case >>>>>> + workdir=/tmp/tidb_cdc_test/savepoint + cluster_type=normal + cd /tmp/tidb_cdc_test/savepoint + DEFAULT_PULSAR_HOME=/usr/local/pulsar + pulsar_dir=/usr/local/pulsar ++ cat + mtls_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderTls brokerClientTlsEnabled=true brokerClientTrustCertsFilePath=/tmp/tidb_cdc_test/savepoint/ca.cert.pem brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls brokerClientAuthenticationParameters={"tlsCertFile":"/tmp/tidb_cdc_test/savepoint/broker_client.cert.pem","tlsKeyFile":"/tmp/tidb_cdc_test/savepoint/broker_client.key-pk8.pem"} brokerServicePortTls=6651 webServicePortTls=8443 tlsTrustCertsFilePath=/tmp/tidb_cdc_test/savepoint/ca.cert.pem tlsCertificateFilePath=/tmp/tidb_cdc_test/savepoint/server.cert.pem tlsKeyFilePath=/tmp/tidb_cdc_test/savepoint/server.key-pk8.pem tlsRequireTrustedClientCertOnConnect=true tlsAllowInsecureConnection=false tlsCertRefreshCheckDurationSec=300' ++ cat + normal_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' ++ cat + mtls_client_conf=' webServiceUrl=https://localhost:8443/ brokerServiceUrl=pulsar+ssl://localhost:6651/ authPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls authParams=tlsCertFile:/tmp/tidb_cdc_test/savepoint/broker_client.cert.pem,tlsKeyFile:/tmp/tidb_cdc_test/savepoint/broker_client.key-pk8.pem tlsTrustCertsFilePath=/tmp/tidb_cdc_test/savepoint/ca.cert.pem' ++ cat + oauth_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/ authPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 authParams={"privateKey":"/tmp/tidb_cdc_test/savepoint/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"}' ++ cat + oauth_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderToken brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 brokerClientAuthenticationParameters={"privateKey":"file:///tmp/tidb_cdc_test/savepoint/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"} tokenSecretKey=data:;base64,U0poWDM2X0thcFlTeWJCdEpxMzVseFhfQnJyNExSVVJTa203UW1YSkdteThwVUZXOUVJT2NWUVBzeWt6OS1qag==' ++ cat + credential_json=' { "client_id":"1234", "client_secret":"e0KVlA2EiBfjoN13olyZd2kv1KL", "audience":"cdc-api-uri", "issuer_url":"http://localhost:9096", "type": "client_credentials" }' ++ cat + cert_server_conf='[ req ] default_bits = 2048 prompt = no default_md = sha256 distinguished_name = dn [ v3_ext ] authorityKeyIdentifier=keyid,issuer:always basicConstraints=CA:FALSE keyUsage=critical, digitalSignature, keyEncipherment extendedKeyUsage=serverAuth subjectAltName=@alt_names [ dn ] CN = server [ alt_names ] DNS.1 = localhost IP.1 = 127.0.0.1' + echo ' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' + cp /usr/local/pulsar/conf/standalone.conf /tmp/tidb_cdc_test/savepoint/pulsar_standalone.conf + pulsar_port=6650 + '[' normal == mtls ']' + '[' normal == oauth ']' + echo 'no cluster type specified, using default configuration.' no cluster type specified, using default configuration. ++ date + echo '[Fri Apr 4 15:37:35 CST 2025] <<<<<< START pulsar cluster in normal mode in savepoint case >>>>>>' [Fri Apr 4 15:37:35 CST 2025] <<<<<< START pulsar cluster in normal mode in savepoint case >>>>>> + echo 'Waiting for pulsar port to be ready...' Waiting for pulsar port to be ready... + i=0 + nc -z localhost 6650 + /usr/local/pulsar/bin/pulsar standalone --config /tmp/tidb_cdc_test/savepoint/pulsar_standalone.conf -nfw --metadata-dir /tmp/tidb_cdc_test/savepoint/pulsar-metadata --bookkeeper-dir /tmp/tidb_cdc_test/savepoint/pulsar-bookie + i=1 + '[' 1 -gt 20 ']' + sleep 2 table ddl_only_block_related_table.finish_mark not exists for 3-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 07:37:36 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [817 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0e2d1312-9897-49dc-9fdf-0c3a86bab5f0 {"id":"0e2d1312-9897-49dc-9fdf-0c3a86bab5f0","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752253} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffbba960cf 0e2d1312-9897-49dc-9fdf-0c3a86bab5f0 /tidb/cdc/default/default/upstream/7489358867304288092 {"id":7489358867304288092,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0e2d1312-9897-49dc-9fdf-0c3a86bab5f0 {"id":"0e2d1312-9897-49dc-9fdf-0c3a86bab5f0","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752253} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffbba960cf 0e2d1312-9897-49dc-9fdf-0c3a86bab5f0 /tidb/cdc/default/default/upstream/7489358867304288092 {"id":7489358867304288092,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0e2d1312-9897-49dc-9fdf-0c3a86bab5f0 {"id":"0e2d1312-9897-49dc-9fdf-0c3a86bab5f0","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752253} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffbba960cf 0e2d1312-9897-49dc-9fdf-0c3a86bab5f0 /tidb/cdc/default/default/upstream/7489358867304288092 {"id":7489358867304288092,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Fri Apr 4 15:37:36 CST 2025] <<<<<< START Pulsar consumer in sink_hang case >>>>>> check_changefeed_status 127.0.0.1:8300 aabaa296-9c5a-4f0f-b775-eee5eadf9658 normal last_error null + endpoint=127.0.0.1:8300 + changefeed_id=aabaa296-9c5a-4f0f-b775-eee5eadf9658 + expected_state=normal + field=last_error + error_pattern=null ++ curl 127.0.0.1:8300/api/v2/changefeeds/aabaa296-9c5a-4f0f-b775-eee5eadf9658/status % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 86 100 86 0 0 1023 0 --:--:-- --:--:-- --:--:-- 1023 + info='{"state":"normal","resolved_ts":457114191478128643,"checkpoint_ts":457114191478128643}' + echo '{"state":"normal","resolved_ts":457114191478128643,"checkpoint_ts":457114191478128643}' {"state":"normal","resolved_ts":457114191478128643,"checkpoint_ts":457114191478128643} ++ echo '{"state":"normal","resolved_ts":457114191478128643,"checkpoint_ts":457114191478128643}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] + [[ -z last_error ]] ++ echo '{"state":"normal","resolved_ts":457114191478128643,"checkpoint_ts":457114191478128643}' ++ jq -r .last_error.message + error_msg=null + [[ ! null =~ null ]] run task successfully check_changefeed_status 127.0.0.1:8300 aabaa296-9c5a-4f0f-b775-eee5eadf9658 normal last_warning null + endpoint=127.0.0.1:8300 + changefeed_id=aabaa296-9c5a-4f0f-b775-eee5eadf9658 + expected_state=normal + field=last_warning + error_pattern=null ++ curl 127.0.0.1:8300/api/v2/changefeeds/aabaa296-9c5a-4f0f-b775-eee5eadf9658/status % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 86 100 86 0 0 1178 0 --:--:-- --:--:-- --:--:-- 1162 100 86 100 86 0 0 1178 0 --:--:-- --:--:-- --:--:-- 1162 + info='{"state":"normal","resolved_ts":457114191478128643,"checkpoint_ts":457114191478128643}' + echo '{"state":"normal","resolved_ts":457114191478128643,"checkpoint_ts":457114191478128643}' {"state":"normal","resolved_ts":457114191478128643,"checkpoint_ts":457114191478128643} ++ echo '{"state":"normal","resolved_ts":457114191478128643,"checkpoint_ts":457114191478128643}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] + [[ -z last_warning ]] ++ echo '{"state":"normal","resolved_ts":457114191478128643,"checkpoint_ts":457114191478128643}' ++ jq -r .last_warning.message + error_msg=null + [[ ! null =~ null ]] run task successfully table sink_hang.t1 not exists for 1-th check, retry later + nc -z localhost 6650 + i=2 + '[' 2 -gt 20 ']' + sleep 2 + nc -z localhost 6650 + i=2 + '[' 2 -gt 20 ']' + sleep 2 table ddl_only_block_related_table.finish_mark not exists for 4-th check, retry later Verifying downstream PD is started... + nc -z localhost 6650 table sink_hang.t1 exists table sink_hang.t2 not exists for 1-th check, retry later + nc -z localhost 6650 + echo 'Waiting for pulsar namespace to be ready...' Waiting for pulsar namespace to be ready... + i=0 + /usr/local/pulsar/bin/pulsar-admin namespaces list public table ddl_only_block_related_table.finish_mark exists + echo 'Waiting for pulsar namespace to be ready...' Waiting for pulsar namespace to be ready... + i=0 + /usr/local/pulsar/bin/pulsar-admin namespaces list public Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release wait process 9540 exit for 1-th time... wait process 9540 exit for 2-th time... /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (9540) - No such process wait process 9540 exit for 3-th time... process 9540 already exit [Fri Apr 4 15:37:40 CST 2025] <<<<<< START cdc server in ddl_only_block_related_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ExecuteNotDone=return(true)' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_only_block_related_table.1014610148.out server --log-file /tmp/tidb_cdc_test/ddl_only_block_related_table/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_only_block_related_table/cdc_data --cluster-id default + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table sink_hang.t2 exists public/default public/default check diff failed 1-th time, retry later Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ++ date + echo '[Fri Apr 4 15:37:41 CST 2025] <<<<<< pulsar is ready >>>>>>' [Fri Apr 4 15:37:41 CST 2025] <<<<<< pulsar is ready >>>>>> + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.savepoint.cli.17073.out cli changefeed create --start-ts=457114190079590401 '--sink-uri=pulsar://127.0.0.1:6650/ticdc-savepoint-test-8808?protocol=canal-json&enable-tidb-extension=true' ++ date + echo '[Fri Apr 4 15:37:41 CST 2025] <<<<<< pulsar is ready >>>>>>' [Fri Apr 4 15:37:41 CST 2025] <<<<<< pulsar is ready >>>>>> + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.move_table.cli.26692.out cli changefeed create --start-ts=457114189666189313 '--sink-uri=pulsar://127.0.0.1:6650/ticdc-move-table-test-31121?protocol=canal-json&enable-tidb-extension=true' Create changefeed successfully! ID: ef21c59c-c09b-455f-8bed-498e9cadbed1 Info: {"upstream_id":7489358853626312113,"namespace":"default","id":"ef21c59c-c09b-455f-8bed-498e9cadbed1","sink_uri":"pulsar://127.0.0.1:6650/ticdc-savepoint-test-8808?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2025-04-04T15:37:42.685044342+08:00","start_ts":457114190079590401,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"pulsar_config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-10-ga306f4894","resolved_ts":457114190079590401,"checkpoint_ts":457114190079590401,"checkpoint_time":"2025-04-04 15:37:30.975"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... Create changefeed successfully! ID: 7e24e3d2-042e-46cc-ac32-4b8248f899b8 Info: {"upstream_id":7489358746991142230,"namespace":"default","id":"7e24e3d2-042e-46cc-ac32-4b8248f899b8","sink_uri":"pulsar://127.0.0.1:6650/ticdc-move-table-test-31121?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2025-04-04T15:37:42.74655459+08:00","start_ts":457114189666189313,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"pulsar_config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-10-ga306f4894","resolved_ts":457114189666189313,"checkpoint_ts":457114189666189313,"checkpoint_time":"2025-04-04 15:37:29.398"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... check diff failed 2-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 07:37:43 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3971 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/ddl-only-block-related-table {UpstreamID:7489358799122364535 Namespace:default ID:ddl-only-block-related-table SinkURI:pulsar://127.0.0.1:6650/ticdc-common-1-test-22617?protocol=canal-json&enable-tidb-extension=true CreateTime:2025-04-04 15:37:29.729663533 +0800 CST StartTs:457114189605371907 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0043123f0 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-10-ga306f4894 Epoch:457114189749288962} {CheckpointTs:457114193235804166 MinTableBarrierTs:457114193235804166 AdminJobType:noop} span: {tableID:118, startKey:7480000000000000ff765f720000000000fa, endKey:7480000000000000ff765f730000000000fa}, resolvedTs: 457114193235804166, checkpointTs: 457114193235804166, state: Replicating span: {tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}, resolvedTs: 457114193235804166, checkpointTs: 457114193235804166, state: Replicating span: {tableID:116, startKey:7480000000000000ff745f720000000000fa, endKey:7480000000000000ff745f730000000000fa}, resolvedTs: 457114193235804166, checkpointTs: 457114193235804166, state: Replicating span: {tableID:120, startKey:7480000000000000ff785f720000000000fa, endKey:7480000000000000ff785f730000000000fa}, resolvedTs: 457114193235804166, checkpointTs: 457114193235804166, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/217ba74e-61b7-4a7a-8134-45c6f9d4615a {"id":"217ba74e-61b7-4a7a-8134-45c6f9d4615a","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752261} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffbb6fcac5 217ba74e-61b7-4a7a-8134-45c6f9d4615a /tidb/cdc/default/default/changefeed/info/ddl-only-block-related-table {"upstream-id":7489358799122364535,"namespace":"default","changefeed-id":"ddl-only-block-related-table","sink-uri":"pulsar://127.0.0.1:6650/ticdc-common-1-test-22617?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2025-04-04T15:37:29.729663533+08:00","start-ts":457114189605371907,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"pulsar-config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-10-ga306f4894","epoch":457114189749288962} /tidb/cdc/default/default/changefeed/status/ddl-only-block-related-table {"checkpoint-ts":457114193235804166,"min-table-barrier-ts":457114193235804166,"admin-job-type":0} /tidb/cdc/default/default/task/position/217ba74e-61b7-4a7a-8134-45c6f9d4615a/ddl-only-block-related-table {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489358799122364535 {"id":7489358799122364535,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-only-block-related-table {UpstreamID:7489358799122364535 Namespace:default ID:ddl-only-block-related-table SinkURI:pulsar://127.0.0.1:6650/ticdc-common-1-test-22617?protocol=canal-json&enable-tidb-extension=true CreateTime:2025-04-04 15:37:29.729663533 +0800 CST StartTs:457114189605371907 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0043123f0 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-10-ga306f4894 Epoch:457114189749288962} {CheckpointTs:457114193235804166 MinTableBarrierTs:457114193235804166 AdminJobType:noop} span: {tableID:118, startKey:7480000000000000ff765f720000000000fa, endKey:7480000000000000ff765f730000000000fa}, resolvedTs: 457114193235804166, checkpointTs: 457114193235804166, state: Replicating span: {tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}, resolvedTs: 457114193235804166, checkpointTs: 457114193235804166, state: Replicating span: {tableID:116, startKey:7480000000000000ff745f720000000000fa, endKey:7480000000000000ff745f730000000000fa}, resolvedTs: 457114193235804166, checkpointTs: 457114193235804166, state: Replicating span: {tableID:120, startKey:7480000000000000ff785f720000000000fa, endKey:7480000000000000ff785f730000000000fa}, resolvedTs: 457114193235804166, checkpointTs: 457114193235804166, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/217ba74e-61b7-4a7a-8134-45c6f9d4615a {"id":"217ba74e-61b7-4a7a-8134-45c6f9d4615a","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752261} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffbb6fcac5 217ba74e-61b7-4a7a-8134-45c6f9d4615a /tidb/cdc/default/default/changefeed/info/ddl-only-block-related-table {"upstream-id":7489358799122364535,"namespace":"default","changefeed-id":"ddl-only-block-related-table","sink-uri":"pulsar://127.0.0.1:6650/ticdc-common-1-test-22617?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2025-04-04T15:37:29.729663533+08:00","start-ts":457114189605371907,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"pulsar-config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-10-ga306f4894","epoch":457114189749288962} /tidb/cdc/default/default/changefeed/status/ddl-only-block-related-table {"checkpoint-ts":457114193235804166,"min-table-barrier-ts":457114193235804166,"admin-job-type":0} /tidb/cdc/default/default/task/position/217ba74e-61b7-4a7a-8134-45c6f9d4615a/ddl-only-block-related-table {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489358799122364535 {"id":7489358799122364535,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-only-block-related-table {UpstreamID:7489358799122364535 Namespace:default ID:ddl-only-block-related-table SinkURI:pulsar://127.0.0.1:6650/ticdc-common-1-test-22617?protocol=canal-json&enable-tidb-extension=true CreateTime:2025-04-04 15:37:29.729663533 +0800 CST StartTs:457114189605371907 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0043123f0 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-10-ga306f4894 Epoch:457114189749288962} {CheckpointTs:457114193235804166 MinTableBarrierTs:457114193235804166 AdminJobType:noop} span: {tableID:118, startKey:7480000000000000ff765f720000000000fa, endKey:7480000000000000ff765f730000000000fa}, resolvedTs: 457114193235804166, checkpointTs: 457114193235804166, state: Replicating span: {tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}, resolvedTs: 457114193235804166, checkpointTs: 457114193235804166, state: Replicating span: {tableID:116, startKey:7480000000000000ff745f720000000000fa, endKey:7480000000000000ff745f730000000000fa}, resolvedTs: 457114193235804166, checkpointTs: 457114193235804166, state: Replicating span: {tableID:120, startKey:7480000000000000ff785f720000000000fa, endKey:7480000000000000ff785f730000000000fa}, resolvedTs: 457114193235804166, checkpointTs: 457114193235804166, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/217ba74e-61b7-4a7a-8134-45c6f9d4615a {"id":"217ba74e-61b7-4a7a-8134-45c6f9d4615a","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752261} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffbb6fcac5 217ba74e-61b7-4a7a-8134-45c6f9d4615a /tidb/cdc/default/default/changefeed/info/ddl-only-block-related-table {"upstream-id":7489358799122364535,"namespace":"default","changefeed-id":"ddl-only-block-related-table","sink-uri":"pulsar://127.0.0.1:6650/ticdc-common-1-test-22617?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2025-04-04T15:37:29.729663533+08:00","start-ts":457114189605371907,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"pulsar-config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-10-ga306f4894","epoch":457114189749288962} /tidb/cdc/default/default/changefeed/status/ddl-only-block-related-table {"checkpoint-ts":457114193235804166,"min-table-barrier-ts":457114193235804166,"admin-job-type":0} /tidb/cdc/default/default/task/position/217ba74e-61b7-4a7a-8134-45c6f9d4615a/ddl-only-block-related-table {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489358799122364535 {"id":7489358799122364535,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x check_ts_not_forward ddl-only-block-related-table + set +x [Fri Apr 4 15:37:44 CST 2025] <<<<<< START Pulsar consumer in savepoint case >>>>>> table savepoint.finish_mark not exists for 1-th check, retry later + set +x [Fri Apr 4 15:37:44 CST 2025] <<<<<< START Pulsar consumer in move_table case >>>>>> [Fri Apr 4 15:37:44 CST 2025] <<<<<< START cdc server in move_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.move_table.2676526771.out server --log-file /tmp/tidb_cdc_test/move_table/cdc2.log --log-level debug --data-dir /tmp/tidb_cdc_test/move_table/cdc_data2 --cluster-id default --addr 127.0.0.1:8301 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8301 failed: Connection refused * Failed to connect to 127.0.0.1 port 8301: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489358765073995196 TiDB Cluster ID. tikv_gc_leader_uuid 657feed9390000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-b84s-w4c6p, pid:10971, start at 2025-04-04 15:37:06.064460179 +0800 CST m=+1.567529247 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:39:06.072 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:37:06.070 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:27:06.070 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489358765073995196 TiDB Cluster ID. tikv_gc_leader_uuid 657feed9390000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-b84s-w4c6p, pid:10971, start at 2025-04-04 15:37:06.064460179 +0800 CST m=+1.567529247 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:39:06.072 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:37:06.070 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:27:06.070 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489358753973086584 TiDB Cluster ID. tikv_gc_leader_uuid 657feed94540003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-b84s-w4c6p, pid:11047, start at 2025-04-04 15:37:06.072486896 +0800 CST m=+1.500698448 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:39:06.083 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:37:06.069 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:27:06.069 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 2 times to try to start tidb cluster... \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-3484/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } check diff successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... table savepoint.finish_mark not exists for 2-th check, retry later cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 4 15:37:47 CST 2025] <<<<<< run test case sink_hang success! >>>>>> + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8301 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 07:37:47 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3971 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/7e24e3d2-042e-46cc-ac32-4b8248f899b8 {UpstreamID:7489358746991142230 Namespace:default ID:7e24e3d2-042e-46cc-ac32-4b8248f899b8 SinkURI:pulsar://127.0.0.1:6650/ticdc-move-table-test-31121?protocol=canal-json&enable-tidb-extension=true CreateTime:2025-04-04 15:37:42.74655459 +0800 CST StartTs:457114189666189313 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003a401b0 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-10-ga306f4894 Epoch:457114193152704526} {CheckpointTs:457114190059667628 MinTableBarrierTs:457114194188435465 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/50882586-7640-49ec-a693-a7ab689247af {"id":"50882586-7640-49ec-a693-a7ab689247af","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752264} /tidb/cdc/default/__cdc_meta__/capture/abe8d4f2-261b-4c68-80b4-55d1683d6a7a {"id":"abe8d4f2-261b-4c68-80b4-55d1683d6a7a","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752252} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffbb399f5f abe8d4f2-261b-4c68-80b4-55d1683d6a7a /tidb/cdc/default/__cdc_meta__/owner/223195ffbb399fdf 50882586-7640-49ec-a693-a7ab689247af /tidb/cdc/default/default/changefeed/info/7e24e3d2-042e-46cc-ac32-4b8248f899b8 {"upstream-id":7489358746991142230,"namespace":"default","changefeed-id":"7e24e3d2-042e-46cc-ac32-4b8248f899b8","sink-uri":"pulsar://127.0.0.1:6650/ticdc-move-table-test-31121?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2025-04-04T15:37:42.74655459+08:00","start-ts":457114189666189313,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"pulsar-config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-10-ga306f4894","epoch":457114193152704526} /tidb/cdc/default/default/changefeed/status/7e24e3d2-042e-46cc-ac32-4b8248f899b8 {"checkpoint-ts":457114190059667628,"min-table-barrier-ts":457114194188435465,"admin-job-type":0} /tidb/cdc/default/default/task/position/50882586-7640-49ec-a693-a7ab689247af/7e24e3d2-042e-46cc-ac32-4b8248f899b8 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/abe8d4f2-261b-4c68-80b4-55d1683d6a7a/7e24e3d2-042e-46cc-ac32-4b8248f899b8 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489358746991142230 {"id":7489358746991142230,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/7e24e3d2-042e-46cc-ac32-4b8248f899b8 {UpstreamID:7489358746991142230 Namespace:default ID:7e24e3d2-042e-46cc-ac32-4b8248f899b8 SinkURI:pulsar://127.0.0.1:6650/ticdc-move-table-test-31121?protocol=canal-json&enable-tidb-extension=true CreateTime:2025-04-04 15:37:42.74655459 +0800 CST StartTs:457114189666189313 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003a401b0 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-10-ga306f4894 Epoch:457114193152704526} {CheckpointTs:457114190059667628 MinTableBarrierTs:457114194188435465 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/50882586-7640-49ec-a693-a7ab689247af {"id":"50882586-7640-49ec-a693-a7ab689247af","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752264} /tidb/cdc/default/__cdc_meta__/capture/abe8d4f2-261b-4c68-80b4-55d1683d6a7a {"id":"abe8d4f2-261b-4c68-80b4-55d1683d6a7a","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752252} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffbb399f5f abe8d4f2-261b-4c68-80b4-55d1683d6a7a /tidb/cdc/default/__cdc_meta__/owner/223195ffbb399fdf 50882586-7640-49ec-a693-a7ab689247af /tidb/cdc/default/default/changefeed/info/7e24e3d2-042e-46cc-ac32-4b8248f899b8 {"upstream-id":7489358746991142230,"namespace":"default","changefeed-id":"7e24e3d2-042e-46cc-ac32-4b8248f899b8","sink-uri":"pulsar://127.0.0.1:6650/ticdc-move-table-test-31121?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2025-04-04T15:37:42.74655459+08:00","start-ts":457114189666189313,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"pulsar-config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-10-ga306f4894","epoch":457114193152704526} /tidb/cdc/default/default/changefeed/status/7e24e3d2-042e-46cc-ac32-4b8248f899b8 {"checkpoint-ts":457114190059667628,"min-table-barrier-ts":457114194188435465,"admin-job-type":0} /tidb/cdc/default/default/task/position/50882586-7640-49ec-a693-a7ab689247af/7e24e3d2-042e-46cc-ac32-4b8248f899b8 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/abe8d4f2-261b-4c68-80b4-55d1683d6a7a/7e24e3d2-042e-46cc-ac32-4b8248f899b8 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489358746991142230 + grep -q 'failed to get info:' {"id":7489358746991142230,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/7e24e3d2-042e-46cc-ac32-4b8248f899b8 {UpstreamID:7489358746991142230 Namespace:default ID:7e24e3d2-042e-46cc-ac32-4b8248f899b8 SinkURI:pulsar://127.0.0.1:6650/ticdc-move-table-test-31121?protocol=canal-json&enable-tidb-extension=true CreateTime:2025-04-04 15:37:42.74655459 +0800 CST StartTs:457114189666189313 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003a401b0 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-10-ga306f4894 Epoch:457114193152704526} {CheckpointTs:457114190059667628 MinTableBarrierTs:457114194188435465 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/50882586-7640-49ec-a693-a7ab689247af {"id":"50882586-7640-49ec-a693-a7ab689247af","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752264} /tidb/cdc/default/__cdc_meta__/capture/abe8d4f2-261b-4c68-80b4-55d1683d6a7a {"id":"abe8d4f2-261b-4c68-80b4-55d1683d6a7a","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752252} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffbb399f5f abe8d4f2-261b-4c68-80b4-55d1683d6a7a /tidb/cdc/default/__cdc_meta__/owner/223195ffbb399fdf 50882586-7640-49ec-a693-a7ab689247af /tidb/cdc/default/default/changefeed/info/7e24e3d2-042e-46cc-ac32-4b8248f899b8 {"upstream-id":7489358746991142230,"namespace":"default","changefeed-id":"7e24e3d2-042e-46cc-ac32-4b8248f899b8","sink-uri":"pulsar://127.0.0.1:6650/ticdc-move-table-test-31121?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2025-04-04T15:37:42.74655459+08:00","start-ts":457114189666189313,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"pulsar-config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-10-ga306f4894","epoch":457114193152704526} /tidb/cdc/default/default/changefeed/status/7e24e3d2-042e-46cc-ac32-4b8248f899b8 {"checkpoint-ts":457114190059667628,"min-table-barrier-ts":457114194188435465,"admin-job-type":0} /tidb/cdc/default/default/task/position/50882586-7640-49ec-a693-a7ab689247af/7e24e3d2-042e-46cc-ac32-4b8248f899b8 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/abe8d4f2-261b-4c68-80b4-55d1683d6a7a/7e24e3d2-042e-46cc-ac32-4b8248f899b8 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} + grep -q 'etcd info' /tidb/cdc/default/default/upstream/7489358746991142230 {"id":7489358746991142230,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x [Fri Apr 4 15:37:47 CST 2025] <<<<<< START cdc server in move_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.move_table.2686926871.out server --log-file /tmp/tidb_cdc_test/move_table/cdc3.log --log-level debug --data-dir /tmp/tidb_cdc_test/move_table/cdc_data3 --cluster-id default --addr 127.0.0.1:8302 + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8302 failed: Connection refused * Failed to connect to 127.0.0.1 port 8302: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 run task failed 1-th time, retry later table savepoint.finish_mark exists check diff successfully wait process cdc.test exit for 1-th time... ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 221 100 221 0 0 7129 0 --:--:-- --:--:-- --:--:-- 7129 + synced_status='{"synced":true,"sink_checkpoint_ts":"2025-04-04 15:37:36.836","puller_resolved_ts":"2025-04-04 15:37:29.786","last_synced_ts":"2025-04-04 15:35:20.785","now_ts":"2025-04-04 15:37:37.000","info":"Data syncing is finished"}' ++ echo '{"synced":true,"sink_checkpoint_ts":"2025-04-04' '15:37:36.836","puller_resolved_ts":"2025-04-04' '15:37:29.786","last_synced_ts":"2025-04-04' '15:35:20.785","now_ts":"2025-04-04' '15:37:37.000","info":"Data' syncing is 'finished"}' ++ jq .synced + status=true + '[' true '!=' true ']' + kill_pd ++ ps aux ++ grep pd-server ++ grep /tmp/tidb_cdc_test/synced_status_with_redo + info='jenkins 11149 7.6 0.0 14967748 145464 ? Sl 15:35 0:11 pd-server --advertise-client-urls http://127.0.0.1:2379 --client-urls http://0.0.0.0:2379 --advertise-peer-urls http://127.0.0.1:2380 --peer-urls http://0.0.0.0:2380 --config /tmp/tidb_cdc_test/synced_status_with_redo/pd-config.toml --log-file /tmp/tidb_cdc_test/synced_status_with_redo/pd1.log --data-dir /tmp/tidb_cdc_test/synced_status_with_redo/pd1 --name=pd1 --initial-cluster=pd1=http://127.0.0.1:2380 jenkins 11203 5.8 0.0 14968260 142644 ? Sl 15:35 0:08 pd-server --advertise-client-urls http://127.0.0.1:2479 --client-urls http://0.0.0.0:2479 --advertise-peer-urls http://127.0.0.1:2480 --peer-urls http://0.0.0.0:2480 --config /tmp/tidb_cdc_test/synced_status_with_redo/pd-config.toml --log-file /tmp/tidb_cdc_test/synced_status_with_redo/down_pd.log --data-dir /tmp/tidb_cdc_test/synced_status_with_redo/down_pd' ++ ps aux ++ grep pd-server ++ grep /tmp/tidb_cdc_test/synced_status_with_redo ++ awk '{print $2}' ++ xargs kill -9 + sleep 20 {"level":"warn","ts":"2025-04-04T15:37:44.679046+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000904e00/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":"2025-04-04T15:37:44.679089+0800","logger":"etcd-client","caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":"2025-04-04T15:37:44.684095+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc002081180/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":"2025-04-04T15:37:44.684135+0800","logger":"etcd-client","caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":"2025-04-04T15:37:44.691367+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc002081180/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2025-04-04T15:37:44.692405+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000904e00/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2025-04-04T15:37:44.820191+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc002100c40/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} {"level":"info","ts":"2025-04-04T15:37:44.820259+0800","logger":"etcd-client","caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":"2025-04-04T15:37:44.830479+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc002100c40/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} {"level":"warn","ts":1743752266.433073,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0048a2540/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":1743752266.4331264,"caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":1743752266.4380941,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc003934700/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":1743752266.4381275,"caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":1743752266.454223,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc003aa4000/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} {"level":"info","ts":1743752266.4542706,"caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 4 15:37:50 CST 2025] <<<<<< run test case savepoint success! >>>>>> {"level":"warn","ts":"2025-04-04T15:37:50.69217+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc002081180/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2025-04-04T15:37:50.69418+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000904e00/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} check_ts_not_forward ddl-only-block-related-table + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8302 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8302 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 07:37:50 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3971 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/7e24e3d2-042e-46cc-ac32-4b8248f899b8 {UpstreamID:7489358746991142230 Namespace:default ID:7e24e3d2-042e-46cc-ac32-4b8248f899b8 SinkURI:pulsar://127.0.0.1:6650/ticdc-move-table-test-31121?protocol=canal-json&enable-tidb-extension=true CreateTime:2025-04-04 15:37:42.74655459 +0800 CST StartTs:457114189666189313 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0040d2240 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-10-ga306f4894 Epoch:457114193152704526} {CheckpointTs:457114190111834253 MinTableBarrierTs:457114194974605325 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/50882586-7640-49ec-a693-a7ab689247af {"id":"50882586-7640-49ec-a693-a7ab689247af","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752264} /tidb/cdc/default/__cdc_meta__/capture/6f8c2310-fab3-4c61-97f5-3494b74b5db8 {"id":"6f8c2310-fab3-4c61-97f5-3494b74b5db8","address":"127.0.0.1:8302","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752268} /tidb/cdc/default/__cdc_meta__/capture/abe8d4f2-261b-4c68-80b4-55d1683d6a7a {"id":"abe8d4f2-261b-4c68-80b4-55d1683d6a7a","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752252} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffbb399f5f abe8d4f2-261b-4c68-80b4-55d1683d6a7a /tidb/cdc/default/__cdc_meta__/owner/223195ffbb399fdf 50882586-7640-49ec-a693-a7ab689247af /tidb/cdc/default/__cdc_meta__/owner/223195ffbb39a017 6f8c2310-fab3-4c61-97f5-3494b74b5db8 /tidb/cdc/default/default/changefeed/info/7e24e3d2-042e-46cc-ac32-4b8248f899b8 {"upstream-id":7489358746991142230,"namespace":"default","changefeed-id":"7e24e3d2-042e-46cc-ac32-4b8248f899b8","sink-uri":"pulsar://127.0.0.1:6650/ticdc-move-table-test-31121?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2025-04-04T15:37:42.74655459+08:00","start-ts":457114189666189313,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"pulsar-config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-10-ga306f4894","epoch":457114193152704526} /tidb/cdc/default/default/changefeed/status/7e24e3d2-042e-46cc-ac32-4b8248f899b8 {"checkpoint-ts":457114190111834253,"min-table-barrier-ts":457114194974605325,"admin-job-type":0} /tidb/cdc/default/default/task/position/50882586-7640-49ec-a693-a7ab689247af/7e24e3d2-042e-46cc-ac32-4b8248f899b8 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/6f8c2310-fab3-4c61-97f5-3494b74b5db8/7e24e3d2-042e-46cc-ac32-4b8248f899b8 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/abe8d4f2-261b-4c68-80b4-55d1683d6a7a/7e24e3d2-042e-46cc-ac32-4b8248f899b8 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489358746991142230 {"id":7489358746991142230,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/7e24e3d2-042e-46cc-ac32-4b8248f899b8 {UpstreamID:7489358746991142230 Namespace:default ID:7e24e3d2-042e-46cc-ac32-4b8248f899b8 SinkURI:pulsar://127.0.0.1:6650/ticdc-move-table-test-31121?protocol=canal-json&enable-tidb-extension=true CreateTime:2025-04-04 15:37:42.74655459 +0800 CST StartTs:457114189666189313 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0040d2240 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-10-ga306f4894 Epoch:457114193152704526} {CheckpointTs:457114190111834253 MinTableBarrierTs:457114194974605325 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/50882586-7640-49ec-a693-a7ab689247af {"id":"50882586-7640-49ec-a693-a7ab689247af","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752264} /tidb/cdc/default/__cdc_meta__/capture/6f8c2310-fab3-4c61-97f5-3494b74b5db8 {"id":"6f8c2310-fab3-4c61-97f5-3494b74b5db8","address":"127.0.0.1:8302","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752268} /tidb/cdc/default/__cdc_meta__/capture/abe8d4f2-261b-4c68-80b4-55d1683d6a7a {"id":"abe8d4f2-261b-4c68-80b4-55d1683d6a7a","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752252} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffbb399f5f abe8d4f2-261b-4c68-80b4-55d1683d6a7a /tidb/cdc/default/__cdc_meta__/owner/223195ffbb399fdf 50882586-7640-49ec-a693-a7ab689247af /tidb/cdc/default/__cdc_meta__/owner/223195ffbb39a017 6f8c2310-fab3-4c61-97f5-3494b74b5db8 /tidb/cdc/default/default/changefeed/info/7e24e3d2-042e-46cc-ac32-4b8248f899b8 {"upstream-id":7489358746991142230,"namespace":"default","changefeed-id":"7e24e3d2-042e-46cc-ac32-4b8248f899b8","sink-uri":"pulsar://127.0.0.1:6650/ticdc-move-table-test-31121?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2025-04-04T15:37:42.74655459+08:00","start-ts":457114189666189313,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"pulsar-config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-10-ga306f4894","epoch":457114193152704526} /tidb/cdc/default/default/changefeed/status/7e24e3d2-042e-46cc-ac32-4b8248f899b8 {"checkpoint-ts":457114190111834253,"min-table-barrier-ts":457114194974605325,"admin-job-type":0} /tidb/cdc/default/default/task/position/50882586-7640-49ec-a693-a7ab689247af/7e24e3d2-042e-46cc-ac32-4b8248f899b8 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/6f8c2310-fab3-4c61-97f5-3494b74b5db8/7e24e3d2-042e-46cc-ac32-4b8248f899b8 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/abe8d4f2-261b-4c68-80b4-55d1683d6a7a/7e24e3d2-042e-46cc-ac32-4b8248f899b8 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489358746991142230 {"id":7489358746991142230,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: changefeedID: default/7e24e3d2-042e-46cc-ac32-4b8248f899b8 {UpstreamID:7489358746991142230 Namespace:default ID:7e24e3d2-042e-46cc-ac32-4b8248f899b8 SinkURI:pulsar://127.0.0.1:6650/ticdc-move-table-test-31121?protocol=canal-json&enable-tidb-extension=true CreateTime:2025-04-04 15:37:42.74655459 +0800 CST StartTs:457114189666189313 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0040d2240 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-10-ga306f4894 Epoch:457114193152704526} {CheckpointTs:457114190111834253 MinTableBarrierTs:457114194974605325 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/50882586-7640-49ec-a693-a7ab689247af {"id":"50882586-7640-49ec-a693-a7ab689247af","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752264} /tidb/cdc/default/__cdc_meta__/capture/6f8c2310-fab3-4c61-97f5-3494b74b5db8 {"id":"6f8c2310-fab3-4c61-97f5-3494b74b5db8","address":"127.0.0.1:8302","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752268} /tidb/cdc/default/__cdc_meta__/capture/abe8d4f2-261b-4c68-80b4-55d1683d6a7a {"id":"abe8d4f2-261b-4c68-80b4-55d1683d6a7a","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752252} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffbb399f5f abe8d4f2-261b-4c68-80b4-55d1683d6a7a /tidb/cdc/default/__cdc_meta__/owner/223195ffbb399fdf 50882586-7640-49ec-a693-a7ab689247af /tidb/cdc/default/__cdc_meta__/owner/223195ffbb39a017 6f8c2310-fab3-4c61-97f5-3494b74b5db8 /tidb/cdc/default/default/changefeed/info/7e24e3d2-042e-46cc-ac32-4b8248f899b8 {"upstream-id":7489358746991142230,"namespace":"default","changefeed-id":"7e24e3d2-042e-46cc-ac32-4b8248f899b8","sink-uri":"pulsar://127.0.0.1:6650/ticdc-move-table-test-31121?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2025-04-04T15:37:42.74655459+08:00","start-ts":457114189666189313,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"pulsar-config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-10-ga306f4894","epoch":457114193152704526} /tidb/cdc/default/default/changefeed/status/7e24e3d2-042e-46cc-ac32-4b8248f899b8 {"checkpoint-ts":457114190111834253,"min-table-barrier-ts":457114194974605325,"admin-job-type":0} /tidb/cdc/default/default/task/position/50882586-7640-49ec-a693-a7ab689247af/7e24e3d2-042e-46cc-ac32-4b8248f899b8 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/6f8c2310-fab3-4c61-97f5-3494b74b5db8/7e24e3d2-042e-46cc-ac32-4b8248f899b8 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/abe8d4f2-261b-4c68-80b4-55d1683d6a7a/7e24e3d2-042e-46cc-ac32-4b8248f899b8 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489358746991142230 {"id":7489358746991142230,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x table move_table.usertable exists go: downloading github.com/IBM/sarama v1.41.2 go: downloading github.com/xdg/scram v1.0.5 go: downloading github.com/tinylib/msgp v1.1.6 go: downloading github.com/benbjohnson/clock v1.1.0 go: downloading go.etcd.io/etcd/server/v3 v3.5.12 go: downloading gorm.io/gorm v1.25.11 go: downloading github.com/KimMachineGun/automemlimit v0.2.4 go: downloading github.com/go-mysql-org/go-mysql v1.12.0 go: downloading github.com/gavv/monotime v0.0.0-20190418164738-30dba4353424 go: downloading github.com/apache/pulsar-client-go v0.11.0 go: downloading github.com/aws/aws-sdk-go-v2 v1.19.1 go: downloading github.com/edwingeng/deque v0.0.0-20191220032131-8596380dee17 go: downloading github.com/gin-gonic/gin v1.9.1 go: downloading github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5 go: downloading github.com/containerd/cgroups v1.0.4 go: downloading github.com/xdg/stringprep v1.0.3 go: downloading github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 go: downloading github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 go: downloading github.com/stretchr/objx v0.5.2 go: downloading github.com/philhofer/fwd v1.1.1 go: downloading github.com/opencontainers/runtime-spec v1.0.2 go: downloading github.com/godbus/dbus/v5 v5.0.4 go: downloading github.com/cilium/ebpf v0.4.0 go: downloading github.com/sirupsen/logrus v1.9.3 go: downloading github.com/jinzhu/now v1.1.5 go: downloading github.com/jinzhu/inflection v1.0.0 go: downloading github.com/gin-contrib/sse v0.1.0 go: downloading github.com/go-playground/validator/v10 v10.14.0 go: downloading github.com/ugorji/go/codec v1.2.11 go: downloading github.com/pelletier/go-toml/v2 v2.0.8 go: downloading filippo.io/edwards25519 v1.1.0 go: downloading github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 go: downloading github.com/aws/smithy-go v1.13.5 go: downloading github.com/gabriel-vasile/mimetype v1.4.2 go: downloading github.com/go-playground/universal-translator v0.18.1 go: downloading github.com/leodido/go-urn v1.2.4 go: downloading github.com/linkedin/goavro/v2 v2.11.1 go: downloading golang.org/x/mod v0.22.0 go: downloading github.com/99designs/keyring v1.2.1 go: downloading github.com/spaolacci/murmur3 v1.1.0 go: downloading github.com/AthenZ/athenz v1.10.39 go: downloading github.com/jcmturner/gokrb5/v8 v8.4.4 go: downloading github.com/jcmturner/gofork v1.7.6 go: downloading github.com/eapache/queue v1.1.0 go: downloading github.com/hashicorp/go-multierror v1.1.1 go: downloading github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 go: downloading github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 go: downloading github.com/eapache/go-resiliency v1.4.0 go: downloading github.com/go-playground/locales v0.14.1 go: downloading github.com/hashicorp/errwrap v1.0.0 go: downloading github.com/dvsekhvalnov/jose2go v1.5.0 go: downloading github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c go: downloading github.com/mtibben/percent v0.2.1 go: downloading github.com/grpc-ecosystem/grpc-gateway v1.16.0 go: downloading go.etcd.io/bbolt v1.3.9 go: downloading github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75 go: downloading go.etcd.io/etcd/pkg/v3 v3.5.12 go: downloading github.com/soheilhy/cmux v0.1.5 go: downloading go.etcd.io/etcd/raft/v3 v3.5.12 go: downloading github.com/xiang90/probing v0.0.0-20221125231312-a49e3df8f510 go: downloading go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.22.0 go: downloading github.com/jonboulle/clockwork v0.4.0 go: downloading go.opentelemetry.io/otel/sdk v1.24.0 go: downloading sigs.k8s.io/yaml v1.4.0 go: downloading github.com/golang-jwt/jwt/v4 v4.5.1 go: downloading go.etcd.io/etcd/client/v2 v2.305.12 go: downloading github.com/gorilla/websocket v1.5.1 go: downloading go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.22.0 go: downloading github.com/jcmturner/dnsutils/v2 v2.0.0 go: downloading github.com/jcmturner/rpc/v2 v2.0.3 go: downloading github.com/hashicorp/go-uuid v1.0.3 go: downloading github.com/jcmturner/aescts/v2 v2.0.0 go: downloading go.opentelemetry.io/proto/otlp v1.1.0 go: downloading github.com/cenkalti/backoff/v4 v4.2.1 {"level":"warn","ts":"2025-04-04T15:37:50.831152+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc002100c40/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} go: downloading github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1 go: downloading github.com/ardielle/ardielle-go v1.5.2 {"level":"warn","ts":1743752275.5173478,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0048a2540/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":1743752275.5324552,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc003aa4000/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} start tidb cluster in /tmp/tidb_cdc_test/sql_mode Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... {"level":"warn","ts":"2025-04-04T15:37:56.69281+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc002081180/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2025-04-04T15:37:56.695797+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000904e00/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2025-04-04T15:37:56.833116+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc002100c40/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-3484/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } Verifying downstream PD is started... [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/synced_status/run.sh using Sink-Type: pulsar... <<================= +++ dirname /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/synced_status/run.sh ++ cd /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/synced_status ++ pwd + CUR=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/synced_status + source /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/synced_status/../_utils/test_prepare ++ UP_TIDB_HOST=127.0.0.1 ++ UP_TIDB_PORT=4000 ++ UP_TIDB_OTHER_PORT=4001 ++ UP_TIDB_STATUS=10080 ++ UP_TIDB_OTHER_STATUS=10081 ++ DOWN_TIDB_HOST=127.0.0.1 ++ DOWN_TIDB_PORT=3306 ++ DOWN_TIDB_STATUS=20080 ++ TLS_TIDB_HOST=127.0.0.1 ++ TLS_TIDB_PORT=3307 ++ TLS_TIDB_STATUS=30080 ++ UP_PD_HOST_1=127.0.0.1 ++ UP_PD_PORT_1=2379 ++ UP_PD_PEER_PORT_1=2380 ++ UP_PD_HOST_2=127.0.0.1 ++ UP_PD_PORT_2=2679 ++ UP_PD_PEER_PORT_2=2680 ++ UP_PD_HOST_3=127.0.0.1 ++ UP_PD_PORT_3=2779 ++ UP_PD_PEER_PORT_3=2780 ++ DOWN_PD_HOST=127.0.0.1 ++ DOWN_PD_PORT=2479 ++ DOWN_PD_PEER_PORT=2480 ++ TLS_PD_HOST=127.0.0.1 ++ TLS_PD_PORT=2579 ++ TLS_PD_PEER_PORT=2580 ++ UP_TIKV_HOST_1=127.0.0.1 ++ UP_TIKV_PORT_1=20160 ++ UP_TIKV_STATUS_PORT_1=20181 ++ UP_TIKV_HOST_2=127.0.0.1 ++ UP_TIKV_PORT_2=20161 ++ UP_TIKV_STATUS_PORT_2=20182 ++ UP_TIKV_HOST_3=127.0.0.1 ++ UP_TIKV_PORT_3=20162 ++ UP_TIKV_STATUS_PORT_3=20183 ++ DOWN_TIKV_HOST=127.0.0.1 ++ DOWN_TIKV_PORT=21160 ++ DOWN_TIKV_STATUS_PORT=21180 ++ TLS_TIKV_HOST=127.0.0.1 ++ TLS_TIKV_PORT=22160 ++ TLS_TIKV_STATUS_PORT=22180 +++ cat /tmp/tidb_cdc_test/KAFKA_VERSION +++ echo 2.4.1 ++ KAFKA_VERSION=2.4.1 + WORK_DIR=/tmp/tidb_cdc_test/synced_status + CDC_BINARY=cdc.test + SINK_TYPE=pulsar + CDC_COUNT=3 + DB_COUNT=4 + trap stop_tidb_cluster EXIT + run_normal_case_and_unavailable_pd conf/changefeed.toml + rm -rf /tmp/tidb_cdc_test/synced_status + mkdir -p /tmp/tidb_cdc_test/synced_status + start_tidb_cluster --workdir /tmp/tidb_cdc_test/synced_status The 1 times to try to start tidb cluster... [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } 0 0 0 0 0 0 0 0 --:--:-- 0:00:01 --:--:-- 0 Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release 0 0 0 0 0 0 0 0 --:--:-- 0:00:02 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:03 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:04 --:--:-- 0{"level":"warn","ts":"2025-04-04T15:38:02.694182+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc002081180/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2025-04-04T15:38:02.696528+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000904e00/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2025-04-04T15:38:02.834256+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc002100c40/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/synced_status Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... 0 0 0 0 0 0 0 0 --:--:-- 0:00:05 --:--:-- 0 [2025/04/04 15:38:04.311 +08:00] [INFO] [main.go:61] ["table mover started"] [2025/04/04 15:38:04.313 +08:00] [INFO] [main.go:165] ["new cluster initialized"] [2025/04/04 15:38:04.313 +08:00] [DEBUG] [main.go:191] ["retrieved owner ID"] [ownerID=abe8d4f2-261b-4c68-80b4-55d1683d6a7a] [2025/04/04 15:38:04.313 +08:00] [DEBUG] [main.go:198] ["retrieved owner addr"] [ownerAddr=127.0.0.1:8300] [2025/04/04 15:38:04.314 +08:00] [DEBUG] [main.go:209] ["retrieved changefeeds"] [changefeedsError="json: unsupported type: map[model.ChangeFeedID]*mvccpb.KeyValue"] [2025/04/04 15:38:04.423 +08:00] [DEBUG] [main.go:228] ["retrieved processor details"] [changefeed=7e24e3d2-042e-46cc-ac32-4b8248f899b8] [captureID=50882586-7640-49ec-a693-a7ab689247af] [processorDetail="{\"table_ids\":[]}"] [2025/04/04 15:38:04.572 +08:00] [DEBUG] [main.go:228] ["retrieved processor details"] [changefeed=7e24e3d2-042e-46cc-ac32-4b8248f899b8] [captureID=6f8c2310-fab3-4c61-97f5-3494b74b5db8] [processorDetail="{\"table_ids\":[]}"] [2025/04/04 15:38:04.722 +08:00] [DEBUG] [main.go:228] ["retrieved processor details"] [changefeed=7e24e3d2-042e-46cc-ac32-4b8248f899b8] [captureID=abe8d4f2-261b-4c68-80b4-55d1683d6a7a] [processorDetail="{\"table_ids\":[114]}"] [2025/04/04 15:38:04.722 +08:00] [INFO] [main.go:75] ["task status"] [status="{\"50882586-7640-49ec-a693-a7ab689247af\":[],\"6f8c2310-fab3-4c61-97f5-3494b74b5db8\":[],\"abe8d4f2-261b-4c68-80b4-55d1683d6a7a\":[{\"ID\":114,\"Changefeed\":\"7e24e3d2-042e-46cc-ac32-4b8248f899b8\"}]}"] [2025/04/04 15:38:04.723 +08:00] [DEBUG] [main.go:287] ["preparing HTTP API call to owner"] [formStr="cf-id=7e24e3d2-042e-46cc-ac32-4b8248f899b8&target-cp-id=6f8c2310-fab3-4c61-97f5-3494b74b5db8&table-id=114"] [2025/04/04 15:38:04.773 +08:00] [INFO] [main.go:179] ["moved table successful"] [tableID=114] [2025/04/04 15:38:04.773 +08:00] [INFO] [main.go:113] ["all tables are moved"] [sourceCapture=abe8d4f2-261b-4c68-80b4-55d1683d6a7a] [targetCapture=6f8c2310-fab3-4c61-97f5-3494b74b5db8] table move_table.check1 not exists for 1-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489358765073995196 TiDB Cluster ID. tikv_gc_leader_uuid 657feed9390000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-b84s-w4c6p, pid:10971, start at 2025-04-04 15:37:06.064460179 +0800 CST m=+1.567529247 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:39:06.072 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:37:06.070 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:27:06.070 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489358765073995196 TiDB Cluster ID. tikv_gc_leader_uuid 657feed9390000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-b84s-w4c6p, pid:10971, start at 2025-04-04 15:37:06.064460179 +0800 CST m=+1.567529247 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:39:06.072 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:37:06.070 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:27:06.070 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489358753973086584 TiDB Cluster ID. tikv_gc_leader_uuid 657feed94540003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-b84s-w4c6p, pid:11047, start at 2025-04-04 15:37:06.072486896 +0800 CST m=+1.500698448 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:39:06.083 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:37:06.069 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:27:06.069 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... 0 0 0 0 0 0 0 0 --:--:-- 0:00:06 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:07 --:--:-- 0 Verifying downstream PD is started... table move_table.check1 not exists for 2-th check, retry later Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release 0 0 0 0 0 0 0 0 --:--:-- 0:00:08 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:09 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:10 --:--:-- 0{"level":"warn","ts":"2025-04-04T15:38:08.694954+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc002081180/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2025-04-04T15:38:08.697751+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000904e00/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2025-04-04T15:38:08.835208+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc002100c40/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table move_table.check1 not exists for 3-th check, retry later 0 0 0 0 0 0 0 0 --:--:-- 0:00:11 --:--:-- 0 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489359034246379442 TiDB Cluster ID. tikv_gc_leader_uuid 657fef185580003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-qg8g-pl0pp, pid:18257, start at 2025-04-04 15:38:10.649048759 +0800 CST m=+1.583199595 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:40:10.657 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:38:10.646 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:28:10.646 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489359034246379442 TiDB Cluster ID. tikv_gc_leader_uuid 657fef185580003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-qg8g-pl0pp, pid:18257, start at 2025-04-04 15:38:10.649048759 +0800 CST m=+1.583199595 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:40:10.657 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:38:10.646 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:28:10.646 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489359038828556693 TiDB Cluster ID. tikv_gc_leader_uuid 657fef18574000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-qg8g-pl0pp, pid:18325, start at 2025-04-04 15:38:10.698438032 +0800 CST m=+1.570474652 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:40:10.706 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:38:10.703 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:28:10.703 +0800 All versions after safe point can be accessed. (DO NOT EDIT) table move_table.check1 not exists for 4-th check, retry later 0 0 0 0 0 0 0 0 --:--:-- 0:00:12 --:--:-- 0 Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/synced_status/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/synced_status/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash-proxy.toml"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash/db/proxy"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } 0 0 0 0 0 0 0 0 --:--:-- 0:00:13 --:--:-- 0 table move_table.check1 not exists for 5-th check, retry later 0 0 0 0 0 0 0 0 --:--:-- 0:00:14 --:--:-- 0 + cd /tmp/tidb_cdc_test/synced_status ++ run_cdc_cli_tso_query 127.0.0.1 2379 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.19781.out cli tso query --pd=http://127.0.0.1:2379 0 0 0 0 0 0 0 0 --:--:-- 0:00:15 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:16 --:--:-- 0{"level":"warn","ts":"2025-04-04T15:38:14.696401+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc002081180/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2025-04-04T15:38:14.699155+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000904e00/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2025-04-04T15:38:14.836378+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc002100c40/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} table move_table.check1 not exists for 6-th check, retry later + set +x + tso='457114201304858625 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 457114201304858625 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + start_ts=457114201304858625 + run_cdc_server --workdir /tmp/tidb_cdc_test/synced_status --binary cdc.test [Fri Apr 4 15:38:15 CST 2025] <<<<<< START cdc server in synced_status case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.1981319815.out server --log-file /tmp/tidb_cdc_test/synced_status/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/synced_status/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 0 0 0 0 0 0 0 0 --:--:-- 0:00:17 --:--:-- 0 table move_table.check1 not exists for 7-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/sql_mode Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... 0 0 0 0 0 0 0 0 --:--:-- 0:00:18 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:19 --:--:-- 0 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 07:38:18 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [817 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ee8e8587-c57c-4c99-a364-e52274517d73 {"id":"ee8e8587-c57c-4c99-a364-e52274517d73","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752295} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffbc44fcd8 ee8e8587-c57c-4c99-a364-e52274517d73 /tidb/cdc/default/default/upstream/7489359034246379442 {"id":7489359034246379442,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ee8e8587-c57c-4c99-a364-e52274517d73 {"id":"ee8e8587-c57c-4c99-a364-e52274517d73","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752295} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffbc44fcd8 ee8e8587-c57c-4c99-a364-e52274517d73 /tidb/cdc/default/default/upstream/7489359034246379442 {"id":7489359034246379442,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ee8e8587-c57c-4c99-a364-e52274517d73 {"id":"ee8e8587-c57c-4c99-a364-e52274517d73","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752295} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffbc44fcd8 ee8e8587-c57c-4c99-a364-e52274517d73 /tidb/cdc/default/default/upstream/7489359034246379442 {"id":7489359034246379442,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + config_path=conf/changefeed.toml + SINK_URI='mysql://root@127.0.0.1:3306/?max-txn-row=1' + run_cdc_cli changefeed create --start-ts=457114201304858625 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/synced_status/conf/changefeed.toml + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.19875.out cli changefeed create --start-ts=457114201304858625 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/synced_status/conf/changefeed.toml Create changefeed successfully! ID: test-1 Info: {"upstream_id":7489359034246379442,"namespace":"default","id":"test-1","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2025-04-04T15:38:18.748917581+08:00","start_ts":457114201304858625,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":120,"checkpoint_interval":20}},"state":"normal","creator_version":"v9.0.0-beta.1-10-ga306f4894","resolved_ts":457114201304858625,"checkpoint_ts":457114201304858625,"checkpoint_time":"2025-04-04 15:38:13.796"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... 0 0 0 0 0 0 0 0 --:--:-- 0:00:20 --:--:-- 0 table move_table.check1 not exists for 8-th check, retry later 0 0 0 0 0 0 0 0 --:--:-- 0:00:21 --:--:-- 0{"level":"warn","ts":"2025-04-04T15:38:19.679907+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000904e00/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":"2025-04-04T15:38:19.67997+0800","logger":"etcd-client","caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":"2025-04-04T15:38:19.685426+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc002081180/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":"2025-04-04T15:38:19.685471+0800","logger":"etcd-client","caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":"2025-04-04T15:38:19.821793+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc002100c40/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} {"level":"info","ts":"2025-04-04T15:38:19.821835+0800","logger":"etcd-client","caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} + set +x ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 221 100 221 0 0 6314 0 --:--:-- --:--:-- --:--:-- 6500 + synced_status='{"synced":true,"sink_checkpoint_ts":"2025-04-04 15:38:20.096","puller_resolved_ts":"1970-01-01 08:00:00.000","last_synced_ts":"1970-01-01 08:00:00.000","now_ts":"2025-04-04 15:38:20.000","info":"Data syncing is finished"}' ++ echo '{"synced":true,"sink_checkpoint_ts":"2025-04-04' '15:38:20.096","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2025-04-04' '15:38:20.000","info":"Data' syncing is 'finished"}' ++ jq .synced + status=true ++ echo '{"synced":true,"sink_checkpoint_ts":"2025-04-04' '15:38:20.096","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2025-04-04' '15:38:20.000","info":"Data' syncing is 'finished"}' ++ jq -r .sink_checkpoint_ts + sink_checkpoint_ts='2025-04-04 15:38:20.096' ++ echo '{"synced":true,"sink_checkpoint_ts":"2025-04-04' '15:38:20.096","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2025-04-04' '15:38:20.000","info":"Data' syncing is 'finished"}' ++ jq -r .puller_resolved_ts + puller_resolved_ts='1970-01-01 08:00:00.000' ++ echo '{"synced":true,"sink_checkpoint_ts":"2025-04-04' '15:38:20.096","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2025-04-04' '15:38:20.000","info":"Data' syncing is 'finished"}' ++ jq -r .last_synced_ts + last_synced_ts='1970-01-01 08:00:00.000' + '[' true '!=' true ']' + '[' '1970-01-01 08:00:00.000' '!=' '1970-01-01 08:00:00.000' ']' + '[' '1970-01-01 08:00:00.000' '!=' '1970-01-01 08:00:00.000' ']' ++ date '+%Y-%m-%d %H:%M:%S' + current='2025-04-04 15:38:20' + echo 'sink_checkpoint_ts is 2025-04-04' 15:38:20.096 sink_checkpoint_ts is 2025-04-04 15:38:20.096 ++ date -d '2025-04-04 15:38:20.096' +%s + checkpoint_timestamp=1743752300 ++ date -d '2025-04-04 15:38:20' +%s + current_timestamp=1743752300 + '[' 0 -gt 300 ']' + run_sql 'USE TEST;Create table t1(a int primary key, b int);insert into t1 values(1,2);insert into t1 values(2,3);' 0 0 0 0 0 0 0 0 --:--:-- 0:00:22 --:--:-- 0{"level":"warn","ts":"2025-04-04T15:38:20.69748+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc002081180/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2025-04-04T15:38:20.700217+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000904e00/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} + check_table_exists test.t1 127.0.0.1 3306 table test.t1 not exists for 1-th check, retry later {"level":"warn","ts":"2025-04-04T15:38:20.837668+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc002100c40/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} table move_table.check1 not exists for 9-th check, retry later 0 0 0 0 0 0 0 0 --:--:-- 0:00:23 --:--:-- 0{"level":"warn","ts":1743752301.4336941,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0048a2540/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":1743752301.4337335,"caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":1743752301.4390986,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc003934700/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":1743752301.4391336,"caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":1743752301.4553664,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc003aa4000/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} {"level":"info","ts":1743752301.4554052,"caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table move_table.check1 not exists for 10-th check, retry later table test.t1 exists + sleep 5 0 0 0 0 0 0 0 0 --:--:-- 0:00:24 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:25 --:--:-- 0 Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table move_table.check1 exists check diff successfully 0 0 0 0 0 0 0 0 --:--:-- 0:00:26 --:--:-- 0 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489358765073995196 TiDB Cluster ID. tikv_gc_leader_uuid 657feed9390000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-b84s-w4c6p, pid:10971, start at 2025-04-04 15:37:06.064460179 +0800 CST m=+1.567529247 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:39:06.072 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:37:06.070 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:27:06.070 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489358765073995196 TiDB Cluster ID. tikv_gc_leader_uuid 657feed9390000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-b84s-w4c6p, pid:10971, start at 2025-04-04 15:37:06.064460179 +0800 CST m=+1.567529247 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:39:06.072 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:37:06.070 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:27:06.070 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489358753973086584 TiDB Cluster ID. tikv_gc_leader_uuid 657feed94540003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-b84s-w4c6p, pid:11047, start at 2025-04-04 15:37:06.072486896 +0800 CST m=+1.500698448 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:39:06.083 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:37:06.069 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:27:06.069 +0800 All versions after safe point can be accessed. (DO NOT EDIT) 0 0 0 0 0 0 0 0 --:--:-- 0:00:27 --:--:-- 0 ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed + run_sql 'set global sql_mode='\''ANSI_QUOTES'\'';' 127.0.0.1 4000 + run_sql 'set global sql_mode='\''ANSI_QUOTES'\'';' 127.0.0.1 3306 ++ run_cdc_cli_tso_query 127.0.0.1 2379 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sql_mode.cli.16734.out cli tso query --pd=http://127.0.0.1:2379 0 0 0 0 0 0 0 0 --:--:-- 0:00:28 --:--:-- 0{"level":"warn","ts":"2025-04-04T15:38:26.697903+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc002081180/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2025-04-04T15:38:26.701422+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000904e00/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2025-04-04T15:38:26.839469+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc002100c40/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} 0 0 0 0 0 0 0 0 --:--:-- 0:00:29 --:--:-- 0 100 135 100 135 0 0 4 0 0:00:33 0:00:30 0:00:03 28 100 135 100 135 0 0 4 0 0:00:33 0:00:30 0:00:03 35 + synced_status='{ "error_msg": "[CDC:ErrPDEtcdAPIError]etcd api call error: context deadline exceeded", "error_code": "CDC:ErrPDEtcdAPIError" }' ++ echo '{' '"error_msg":' '"[CDC:ErrPDEtcdAPIError]etcd' api call error: context deadline 'exceeded",' '"error_code":' '"CDC:ErrPDEtcdAPIError"' '}' ++ jq -r .error_code + error_code=CDC:ErrPDEtcdAPIError + cleanup_process cdc.test wait process cdc.test exit for 1-th time... ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 243 100 243 0 0 40500 0 --:--:-- --:--:-- --:--:-- 40500 + synced_status='{"synced":false,"sink_checkpoint_ts":"2025-04-04 15:38:27.096","puller_resolved_ts":"2025-04-04 15:38:20.246","last_synced_ts":"2025-04-04 15:38:20.746","now_ts":"2025-04-04 15:38:27.000","info":"The data syncing is not finished, please wait"}' ++ echo '{"synced":false,"sink_checkpoint_ts":"2025-04-04' '15:38:27.096","puller_resolved_ts":"2025-04-04' '15:38:20.246","last_synced_ts":"2025-04-04' '15:38:20.746","now_ts":"2025-04-04' '15:38:27.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq .synced + status=false + '[' false '!=' false ']' ++ echo '{"synced":false,"sink_checkpoint_ts":"2025-04-04' '15:38:27.096","puller_resolved_ts":"2025-04-04' '15:38:20.246","last_synced_ts":"2025-04-04' '15:38:20.746","now_ts":"2025-04-04' '15:38:27.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq -r .info + info='The data syncing is not finished, please wait' + '[' 'The data syncing is not finished, please wait' '!=' 'The data syncing is not finished, please wait' ']' + sleep 130 + set +x + tso='457114204789538817 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 457114204789538817 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + start_ts=457114204789538817 + run_cdc_server --workdir /tmp/tidb_cdc_test/sql_mode --binary cdc.test [Fri Apr 4 15:38:28 CST 2025] <<<<<< START cdc server in sql_mode case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sql_mode.1676516767.out server --log-file /tmp/tidb_cdc_test/sql_mode/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/sql_mode/cdc_data --cluster-id default + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 wait process cdc.test exit for 2-th time... [2025/04/04 15:38:29.140 +08:00] [INFO] [main.go:61] ["table mover started"] [2025/04/04 15:38:29.142 +08:00] [INFO] [main.go:165] ["new cluster initialized"] [2025/04/04 15:38:29.142 +08:00] [DEBUG] [main.go:191] ["retrieved owner ID"] [ownerID=abe8d4f2-261b-4c68-80b4-55d1683d6a7a] [2025/04/04 15:38:29.142 +08:00] [DEBUG] [main.go:198] ["retrieved owner addr"] [ownerAddr=127.0.0.1:8300] [2025/04/04 15:38:29.143 +08:00] [DEBUG] [main.go:209] ["retrieved changefeeds"] [changefeedsError="json: unsupported type: map[model.ChangeFeedID]*mvccpb.KeyValue"] [2025/04/04 15:38:29.272 +08:00] [DEBUG] [main.go:228] ["retrieved processor details"] [changefeed=7e24e3d2-042e-46cc-ac32-4b8248f899b8] [captureID=50882586-7640-49ec-a693-a7ab689247af] [processorDetail="{\"table_ids\":[]}"] [2025/04/04 15:38:29.422 +08:00] [DEBUG] [main.go:228] ["retrieved processor details"] [changefeed=7e24e3d2-042e-46cc-ac32-4b8248f899b8] [captureID=6f8c2310-fab3-4c61-97f5-3494b74b5db8] [processorDetail="{\"table_ids\":[118]}"] [2025/04/04 15:38:29.572 +08:00] [DEBUG] [main.go:228] ["retrieved processor details"] [changefeed=7e24e3d2-042e-46cc-ac32-4b8248f899b8] [captureID=abe8d4f2-261b-4c68-80b4-55d1683d6a7a] [processorDetail="{\"table_ids\":[116]}"] [2025/04/04 15:38:29.572 +08:00] [INFO] [main.go:75] ["task status"] [status="{\"50882586-7640-49ec-a693-a7ab689247af\":[],\"6f8c2310-fab3-4c61-97f5-3494b74b5db8\":[{\"ID\":118,\"Changefeed\":\"7e24e3d2-042e-46cc-ac32-4b8248f899b8\"}],\"abe8d4f2-261b-4c68-80b4-55d1683d6a7a\":[{\"ID\":116,\"Changefeed\":\"7e24e3d2-042e-46cc-ac32-4b8248f899b8\"}]}"] [2025/04/04 15:38:29.573 +08:00] [DEBUG] [main.go:287] ["preparing HTTP API call to owner"] [formStr="cf-id=7e24e3d2-042e-46cc-ac32-4b8248f899b8&target-cp-id=abe8d4f2-261b-4c68-80b4-55d1683d6a7a&table-id=118"] [2025/04/04 15:38:29.622 +08:00] [INFO] [main.go:179] ["moved table successful"] [tableID=118] [2025/04/04 15:38:29.622 +08:00] [INFO] [main.go:113] ["all tables are moved"] [sourceCapture=6f8c2310-fab3-4c61-97f5-3494b74b5db8] [targetCapture=abe8d4f2-261b-4c68-80b4-55d1683d6a7a] cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit + stop_tidb_cluster check diff successfully table move_table.check2 not exists for 1-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 07:38:31 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3971 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/test-1 {UpstreamID:7489358765073995196 Namespace:default ID:test-1 SinkURI:mysql://root@127.0.0.1:3306/?max-txn-row=1 CreateTime:2025-04-04 15:37:14.304251527 +0800 CST StartTs:457114184390017025 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0043a6750 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-10-ga306f4894 Epoch:457114185700474886} {CheckpointTs:457114186028154884 MinTableBarrierTs:457114186028154884 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/43dfc674-a146-4638-9ad3-f34f3a58ef08 {"id":"43dfc674-a146-4638-9ad3-f34f3a58ef08","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752231} /tidb/cdc/default/__cdc_meta__/capture/eff47a31-c2e6-4ff4-8401-9cfacf1b3379 {"id":"eff47a31-c2e6-4ff4-8401-9cfacf1b3379","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752308} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffbb46fdd9 43dfc674-a146-4638-9ad3-f34f3a58ef08 /tidb/cdc/default/__cdc_meta__/owner/223195ffbc752dd2 eff47a31-c2e6-4ff4-8401-9cfacf1b3379 /tidb/cdc/default/default/changefeed/info/test-1 {"upstream-id":7489358765073995196,"namespace":"default","changefeed-id":"test-1","sink-uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create-time":"2025-04-04T15:37:14.304251527+08:00","start-ts":457114184390017025,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-10-ga306f4894","epoch":457114185700474886} /tidb/cdc/default/default/changefeed/status/test-1 {"checkpoint-ts":457114186028154884,"min-table-barrier-ts":457114186028154884,"admin-job-type":0} /tidb/cdc/default/default/task/position/43dfc674-a146-4638-9ad3-f34f3a58ef08/test-1 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/eff47a31-c2e6-4ff4-8401-9cfacf1b3379/test-1 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489358765073995196 {"id":7489358765073995196,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/test-1 {UpstreamID:7489358765073995196 Namespace:default ID:test-1 SinkURI:mysql://root@127.0.0.1:3306/?max-txn-row=1 CreateTime:2025-04-04 15:37:14.304251527 +0800 CST StartTs:457114184390017025 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0043a6750 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-10-ga306f4894 Epoch:457114185700474886} {CheckpointTs:457114186028154884 MinTableBarrierTs:457114186028154884 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/43dfc674-a146-4638-9ad3-f34f3a58ef08 {"id":"43dfc674-a146-4638-9ad3-f34f3a58ef08","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752231} /tidb/cdc/default/__cdc_meta__/capture/eff47a31-c2e6-4ff4-8401-9cfacf1b3379 {"id":"eff47a31-c2e6-4ff4-8401-9cfacf1b3379","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752308} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffbb46fdd9 43dfc674-a146-4638-9ad3-f34f3a58ef08 /tidb/cdc/default/__cdc_meta__/owner/223195ffbc752dd2 eff47a31-c2e6-4ff4-8401-9cfacf1b3379 /tidb/cdc/default/default/changefeed/info/test-1 {"upstream-id":7489358765073995196,"namespace":"default","changefeed-id":"test-1","sink-uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create-time":"2025-04-04T15:37:14.304251527+08:00","start-ts":457114184390017025,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-10-ga306f4894","epoch":457114185700474886} /tidb/cdc/default/default/changefeed/status/test-1 {"checkpoint-ts":457114186028154884,"min-table-barrier-ts":457114186028154884,"admin-job-type":0} /tidb/cdc/default/default/task/position/43dfc674-a146-4638-9ad3-f34f3a58ef08/test-1 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/eff47a31-c2e6-4ff4-8401-9cfacf1b3379/test-1 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489358765073995196 {"id":7489358765073995196,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: changefeedID: default/test-1 {UpstreamID:7489358765073995196 Namespace:default ID:test-1 SinkURI:mysql://root@127.0.0.1:3306/?max-txn-row=1 CreateTime:2025-04-04 15:37:14.304251527 +0800 CST StartTs:457114184390017025 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0043a6750 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-10-ga306f4894 Epoch:457114185700474886} {CheckpointTs:457114186028154884 MinTableBarrierTs:457114186028154884 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/43dfc674-a146-4638-9ad3-f34f3a58ef08 {"id":"43dfc674-a146-4638-9ad3-f34f3a58ef08","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752231} /tidb/cdc/default/__cdc_meta__/capture/eff47a31-c2e6-4ff4-8401-9cfacf1b3379 {"id":"eff47a31-c2e6-4ff4-8401-9cfacf1b3379","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752308} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffbb46fdd9 43dfc674-a146-4638-9ad3-f34f3a58ef08 /tidb/cdc/default/__cdc_meta__/owner/223195ffbc752dd2 eff47a31-c2e6-4ff4-8401-9cfacf1b3379 /tidb/cdc/default/default/changefeed/info/test-1 {"upstream-id":7489358765073995196,"namespace":"default","changefeed-id":"test-1","sink-uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create-time":"2025-04-04T15:37:14.304251527+08:00","start-ts":457114184390017025,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-10-ga306f4894","epoch":457114185700474886} /tidb/cdc/default/default/changefeed/status/test-1 {"checkpoint-ts":457114186028154884,"min-table-barrier-ts":457114186028154884,"admin-job-type":0} /tidb/cdc/default/default/task/position/43dfc674-a146-4638-9ad3-f34f3a58ef08/test-1 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/eff47a31-c2e6-4ff4-8401-9cfacf1b3379/test-1 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489358765073995196 {"id":7489358765073995196,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + SINK_URI='mysql://root@127.0.0.1:3306/?max-txn-row=1' + run_cdc_cli changefeed create --start-ts=457114204789538817 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-2 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sql_mode.cli.16827.out cli changefeed create --start-ts=457114204789538817 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-2 table move_table.check2 exists check diff successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Fri Apr 4 15:38:34 CST 2025] <<<<<< run test case move_table success! >>>>>> Create changefeed successfully! ID: test-2 Info: {"upstream_id":7489358765073995196,"namespace":"default","id":"test-2","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2025-04-04T15:38:34.346421399+08:00","start_ts":457114204789538817,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-10-ga306f4894","resolved_ts":457114204789538817,"checkpoint_ts":457114204789538817,"checkpoint_time":"2025-04-04 15:38:27.089"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x + run_sql 'use test; create table t2(id bigint primary key, a date); insert into t2 values(1, '\''2023-02-08'\'');' 127.0.0.1 4000 pass check, checkpoint tso not forward after 10s run task successfully + run_case_with_unavailable_tikv conf/changefeed-redo.toml + rm -rf /tmp/tidb_cdc_test/synced_status_with_redo + mkdir -p /tmp/tidb_cdc_test/synced_status_with_redo + start_tidb_cluster --workdir /tmp/tidb_cdc_test/synced_status_with_redo shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory The 1 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory wait process 10151 exit for 1-th time... wait process 10151 exit for 2-th time... wait process 10151 exit for 3-th time... wait process 10151 exit for 4-th time... /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (10151) - No such process wait process 10151 exit for 5-th time... process 10151 already exit [Fri Apr 4 15:38:39 CST 2025] <<<<<< START cdc server in ddl_only_block_related_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_only_block_related_table.1075410756.out server --log-file /tmp/tidb_cdc_test/ddl_only_block_related_table/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_only_block_related_table/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/synced_status_with_redo Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 07:38:43 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [3971 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/ddl-only-block-related-table {UpstreamID:7489358799122364535 Namespace:default ID:ddl-only-block-related-table SinkURI:pulsar://127.0.0.1:6650/ticdc-common-1-test-22617?protocol=canal-json&enable-tidb-extension=true CreateTime:2025-04-04 15:37:29.729663533 +0800 CST StartTs:457114189605371907 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00354ba70 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-10-ga306f4894 Epoch:457114189749288962} {CheckpointTs:457114193497948206 MinTableBarrierTs:457114208728776705 AdminJobType:noop} span: {tableID:118, startKey:7480000000000000ff765f720000000000fa, endKey:7480000000000000ff765f730000000000fa}, resolvedTs: 457114193497948206, checkpointTs: 457114193497948206, state: Preparing span: {tableID:120, startKey:7480000000000000ff785f720000000000fa, endKey:7480000000000000ff785f730000000000fa}, resolvedTs: 457114193497948206, checkpointTs: 457114193497948206, state: Preparing span: {tableID:116, startKey:7480000000000000ff745f720000000000fa, endKey:7480000000000000ff745f730000000000fa}, resolvedTs: 457114193497948206, checkpointTs: 457114193497948206, state: Preparing span: {tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}, resolvedTs: 457114193497948206, checkpointTs: 457114193497948206, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d4fab96f-90d1-46e0-8741-63717cef2334 {"id":"d4fab96f-90d1-46e0-8741-63717cef2334","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752320} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffbb6fccb5 d4fab96f-90d1-46e0-8741-63717cef2334 /tidb/cdc/default/default/changefeed/info/ddl-only-block-related-table {"upstream-id":7489358799122364535,"namespace":"default","changefeed-id":"ddl-only-block-related-table","sink-uri":"pulsar://127.0.0.1:6650/ticdc-common-1-test-22617?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2025-04-04T15:37:29.729663533+08:00","start-ts":457114189605371907,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"pulsar-config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-10-ga306f4894","epoch":457114189749288962} /tidb/cdc/default/default/changefeed/status/ddl-only-block-related-table {"checkpoint-ts":457114193497948206,"min-table-barrier-ts":457114208728776705,"admin-job-type":0} /tidb/cdc/default/default/task/position/d4fab96f-90d1-46e0-8741-63717cef2334/ddl-only-block-related-table {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489358799122364535 {"id":7489358799122364535,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-only-block-related-table {UpstreamID:7489358799122364535 Namespace:default ID:ddl-only-block-related-table SinkURI:pulsar://127.0.0.1:6650/ticdc-common-1-test-22617?protocol=canal-json&enable-tidb-extension=true CreateTime:2025-04-04 15:37:29.729663533 +0800 CST StartTs:457114189605371907 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00354ba70 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-10-ga306f4894 Epoch:457114189749288962} {CheckpointTs:457114193497948206 MinTableBarrierTs:457114208728776705 AdminJobType:noop} span: {tableID:118, startKey:7480000000000000ff765f720000000000fa, endKey:7480000000000000ff765f730000000000fa}, resolvedTs: 457114193497948206, checkpointTs: 457114193497948206, state: Preparing span: {tableID:120, startKey:7480000000000000ff785f720000000000fa, endKey:7480000000000000ff785f730000000000fa}, resolvedTs: 457114193497948206, checkpointTs: 457114193497948206, state: Preparing span: {tableID:116, startKey:7480000000000000ff745f720000000000fa, endKey:7480000000000000ff745f730000000000fa}, resolvedTs: 457114193497948206, checkpointTs: 457114193497948206, state: Preparing span: {tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}, resolvedTs: 457114193497948206, checkpointTs: 457114193497948206, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d4fab96f-90d1-46e0-8741-63717cef2334 {"id":"d4fab96f-90d1-46e0-8741-63717cef2334","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752320} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffbb6fccb5 d4fab96f-90d1-46e0-8741-63717cef2334 /tidb/cdc/default/default/changefeed/info/ddl-only-block-related-table {"upstream-id":7489358799122364535,"namespace":"default","changefeed-id":"ddl-only-block-related-table","sink-uri":"pulsar://127.0.0.1:6650/ticdc-common-1-test-22617?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2025-04-04T15:37:29.729663533+08:00","start-ts":457114189605371907,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"pulsar-config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-10-ga306f4894","epoch":457114189749288962} /tidb/cdc/default/default/changefeed/status/ddl-only-block-related-table {"checkpoint-ts":457114193497948206,"min-table-barrier-ts":457114208728776705,"admin-job-type":0} /tidb/cdc/default/default/task/position/d4fab96f-90d1-46e0-8741-63717cef2334/ddl-only-block-related-table {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489358799122364535 {"id":7489358799122364535,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-only-block-related-table {UpstreamID:7489358799122364535 Namespace:default ID:ddl-only-block-related-table SinkURI:pulsar://127.0.0.1:6650/ticdc-common-1-test-22617?protocol=canal-json&enable-tidb-extension=true CreateTime:2025-04-04 15:37:29.729663533 +0800 CST StartTs:457114189605371907 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00354ba70 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-10-ga306f4894 Epoch:457114189749288962} {CheckpointTs:457114193497948206 MinTableBarrierTs:457114208728776705 AdminJobType:noop} span: {tableID:118, startKey:7480000000000000ff765f720000000000fa, endKey:7480000000000000ff765f730000000000fa}, resolvedTs: 457114193497948206, checkpointTs: 457114193497948206, state: Preparing span: {tableID:120, startKey:7480000000000000ff785f720000000000fa, endKey:7480000000000000ff785f730000000000fa}, resolvedTs: 457114193497948206, checkpointTs: 457114193497948206, state: Preparing span: {tableID:116, startKey:7480000000000000ff745f720000000000fa, endKey:7480000000000000ff745f730000000000fa}, resolvedTs: 457114193497948206, checkpointTs: 457114193497948206, state: Preparing span: {tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}, resolvedTs: 457114193497948206, checkpointTs: 457114193497948206, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d4fab96f-90d1-46e0-8741-63717cef2334 {"id":"d4fab96f-90d1-46e0-8741-63717cef2334","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752320} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffbb6fccb5 d4fab96f-90d1-46e0-8741-63717cef2334 /tidb/cdc/default/default/changefeed/info/ddl-only-block-related-table {"upstream-id":7489358799122364535,"namespace":"default","changefeed-id":"ddl-only-block-related-table","sink-uri":"pulsar://127.0.0.1:6650/ticdc-common-1-test-22617?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2025-04-04T15:37:29.729663533+08:00","start-ts":457114189605371907,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"pulsar-config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-10-ga306f4894","epoch":457114189749288962} /tidb/cdc/default/default/changefeed/status/ddl-only-block-related-table {"checkpoint-ts":457114193497948206,"min-table-barrier-ts":457114208728776705,"admin-job-type":0} /tidb/cdc/default/default/task/position/d4fab96f-90d1-46e0-8741-63717cef2334/ddl-only-block-related-table {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489358799122364535 {"id":7489358799122364535,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x check diff failed 1-th time, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/autorandom/run.sh using Sink-Type: pulsar... <<================= The 1 times to try to start tidb cluster... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release check diff failed 2-th time, retry later Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/autorandom Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... check diff successfully check_ts_forward ddl-only-block-related-table VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489359200616023092 TiDB Cluster ID. tikv_gc_leader_uuid 657fef3d2fc000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-9bld-w6hq9, pid:14945, start at 2025-04-04 15:38:48.403606044 +0800 CST m=+1.491204963 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:40:48.411 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:38:48.383 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:28:48.383 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489359200616023092 TiDB Cluster ID. tikv_gc_leader_uuid 657fef3d2fc000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-9bld-w6hq9, pid:14945, start at 2025-04-04 15:38:48.403606044 +0800 CST m=+1.491204963 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:40:48.411 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:38:48.383 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:28:48.383 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489359191578982048 TiDB Cluster ID. tikv_gc_leader_uuid 657fef3d2e4000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-9bld-w6hq9, pid:15022, start at 2025-04-04 15:38:48.417129865 +0800 CST m=+1.430520346 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:40:48.424 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:38:48.427 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:28:48.427 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/synced_status_with_redo/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/synced_status_with_redo/tiflash/log/error.log arg matches is ArgMatches { args: {"pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/synced_status_with_redo/tiflash/db/proxy"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/synced_status_with_redo/tiflash-proxy.toml"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/synced_status_with_redo/tiflash/log/proxy.log"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } changefeed is working normally rts: 457114210039234561->457114210301378561 checkpoint: 457114210039234561->457114210301378561 run task successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 4 15:38:50 CST 2025] <<<<<< run test case ddl_only_block_related_table success! >>>>>> + cd /tmp/tidb_cdc_test/synced_status_with_redo ++ run_cdc_cli_tso_query 127.0.0.1 2379 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status_with_redo.cli.16341.out cli tso query --pd=http://127.0.0.1:2379 Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x + tso='457114211223339009 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 457114211223339009 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + start_ts=457114211223339009 + run_cdc_server --workdir /tmp/tidb_cdc_test/synced_status_with_redo --binary cdc.test [Fri Apr 4 15:38:53 CST 2025] <<<<<< START cdc server in synced_status_with_redo case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status_with_redo.1637716379.out server --log-file /tmp/tidb_cdc_test/synced_status_with_redo/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/synced_status_with_redo/cdc_data --cluster-id default + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489359227209379325 TiDB Cluster ID. tikv_gc_leader_uuid 657fef42bf80009 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-6nlr-p88d4, pid:32171, start at 2025-04-04 15:38:54.092398658 +0800 CST m=+1.635912148 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:40:54.102 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:38:54.078 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:28:54.078 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489359227209379325 TiDB Cluster ID. tikv_gc_leader_uuid 657fef42bf80009 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-6nlr-p88d4, pid:32171, start at 2025-04-04 15:38:54.092398658 +0800 CST m=+1.635912148 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:40:54.102 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:38:54.078 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:28:54.078 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489359229294435856 TiDB Cluster ID. tikv_gc_leader_uuid 657fef42d8c000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-6nlr-p88d4, pid:32246, start at 2025-04-04 15:38:54.223785572 +0800 CST m=+1.687664548 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:40:54.232 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:38:54.229 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:28:54.229 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/autorandom/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/autorandom/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/autorandom/tiflash/db/proxy"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/autorandom/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/autorandom/tiflash/log/proxy.log"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 07:38:56 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [817 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/355576fe-3fc3-48f1-aa79-5c4ac62a1a17 {"id":"355576fe-3fc3-48f1-aa79-5c4ac62a1a17","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752333} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffbcd707d6 355576fe-3fc3-48f1-aa79-5c4ac62a1a17 /tidb/cdc/default/default/upstream/7489359200616023092 {"id":7489359200616023092,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/355576fe-3fc3-48f1-aa79-5c4ac62a1a17 {"id":"355576fe-3fc3-48f1-aa79-5c4ac62a1a17","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752333} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffbcd707d6 355576fe-3fc3-48f1-aa79-5c4ac62a1a17 /tidb/cdc/default/default/upstream/7489359200616023092 {"id":7489359200616023092,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/355576fe-3fc3-48f1-aa79-5c4ac62a1a17 {"id":"355576fe-3fc3-48f1-aa79-5c4ac62a1a17","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752333} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffbcd707d6 355576fe-3fc3-48f1-aa79-5c4ac62a1a17 /tidb/cdc/default/default/upstream/7489359200616023092 {"id":7489359200616023092,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + config_path=conf/changefeed-redo.toml + SINK_URI='mysql://root@127.0.0.1:3306/?max-txn-row=1' + run_cdc_cli changefeed create --start-ts=457114211223339009 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/synced_status_with_redo/conf/changefeed-redo.toml + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status_with_redo.cli.16439.out cli changefeed create --start-ts=457114211223339009 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/synced_status_with_redo/conf/changefeed-redo.toml Create changefeed successfully! ID: test-1 Info: {"upstream_id":7489359200616023092,"namespace":"default","id":"test-1","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2025-04-04T15:38:56.606966235+08:00","start_ts":457114211223339009,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"eventual","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"storage":"file:///tmp/tidb_cdc_test/synced_status/redo","use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":120,"checkpoint_interval":20}},"state":"normal","creator_version":"v9.0.0-beta.1-10-ga306f4894","resolved_ts":457114211223339009,"checkpoint_ts":457114211223339009,"checkpoint_time":"2025-04-04 15:38:51.632"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... [Fri Apr 4 15:38:56 CST 2025] <<<<<< START cdc server in autorandom case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.autorandom.3353333535.out server --log-file /tmp/tidb_cdc_test/autorandom/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/autorandom/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + set +x + run_sql 'USE TEST;Create table t1(a int primary key, b int);insert into t1 values(1,2);insert into t1 values(2,3);' + check_table_exists test.t1 127.0.0.1 3306 table test.t1 not exists for 1-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 07:38:59 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [817 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/fc6bef28-19b4-4009-bd4d-fa44816573e3 {"id":"fc6bef28-19b4-4009-bd4d-fa44816573e3","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752337} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffbcf099cc fc6bef28-19b4-4009-bd4d-fa44816573e3 /tidb/cdc/default/default/upstream/7489359227209379325 {"id":7489359227209379325,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/fc6bef28-19b4-4009-bd4d-fa44816573e3 {"id":"fc6bef28-19b4-4009-bd4d-fa44816573e3","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752337} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffbcf099cc fc6bef28-19b4-4009-bd4d-fa44816573e3 /tidb/cdc/default/default/upstream/7489359227209379325 {"id":7489359227209379325,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/fc6bef28-19b4-4009-bd4d-fa44816573e3 {"id":"fc6bef28-19b4-4009-bd4d-fa44816573e3","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752337} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffbcf099cc fc6bef28-19b4-4009-bd4d-fa44816573e3 /tidb/cdc/default/default/upstream/7489359227209379325 {"id":7489359227209379325,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x ++ date + echo '[Fri Apr 4 15:39:00 CST 2025] <<<<<< START pulsar cluster in autorandom case >>>>>>' [Fri Apr 4 15:39:00 CST 2025] <<<<<< START pulsar cluster in autorandom case >>>>>> + workdir=/tmp/tidb_cdc_test/autorandom + cluster_type=normal + cd /tmp/tidb_cdc_test/autorandom + DEFAULT_PULSAR_HOME=/usr/local/pulsar + pulsar_dir=/usr/local/pulsar ++ cat + mtls_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderTls brokerClientTlsEnabled=true brokerClientTrustCertsFilePath=/tmp/tidb_cdc_test/autorandom/ca.cert.pem brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls brokerClientAuthenticationParameters={"tlsCertFile":"/tmp/tidb_cdc_test/autorandom/broker_client.cert.pem","tlsKeyFile":"/tmp/tidb_cdc_test/autorandom/broker_client.key-pk8.pem"} brokerServicePortTls=6651 webServicePortTls=8443 tlsTrustCertsFilePath=/tmp/tidb_cdc_test/autorandom/ca.cert.pem tlsCertificateFilePath=/tmp/tidb_cdc_test/autorandom/server.cert.pem tlsKeyFilePath=/tmp/tidb_cdc_test/autorandom/server.key-pk8.pem tlsRequireTrustedClientCertOnConnect=true tlsAllowInsecureConnection=false tlsCertRefreshCheckDurationSec=300' ++ cat + normal_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' ++ cat + mtls_client_conf=' webServiceUrl=https://localhost:8443/ brokerServiceUrl=pulsar+ssl://localhost:6651/ authPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls authParams=tlsCertFile:/tmp/tidb_cdc_test/autorandom/broker_client.cert.pem,tlsKeyFile:/tmp/tidb_cdc_test/autorandom/broker_client.key-pk8.pem tlsTrustCertsFilePath=/tmp/tidb_cdc_test/autorandom/ca.cert.pem' ++ cat + oauth_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/ authPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 authParams={"privateKey":"/tmp/tidb_cdc_test/autorandom/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"}' ++ cat + oauth_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderToken brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 brokerClientAuthenticationParameters={"privateKey":"file:///tmp/tidb_cdc_test/autorandom/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"} tokenSecretKey=data:;base64,U0poWDM2X0thcFlTeWJCdEpxMzVseFhfQnJyNExSVVJTa203UW1YSkdteThwVUZXOUVJT2NWUVBzeWt6OS1qag==' ++ cat + credential_json=' { "client_id":"1234", "client_secret":"e0KVlA2EiBfjoN13olyZd2kv1KL", "audience":"cdc-api-uri", "issuer_url":"http://localhost:9096", "type": "client_credentials" }' ++ cat + cert_server_conf='[ req ] default_bits = 2048 prompt = no default_md = sha256 distinguished_name = dn [ v3_ext ] authorityKeyIdentifier=keyid,issuer:always basicConstraints=CA:FALSE keyUsage=critical, digitalSignature, keyEncipherment extendedKeyUsage=serverAuth subjectAltName=@alt_names [ dn ] CN = server [ alt_names ] DNS.1 = localhost IP.1 = 127.0.0.1' + echo ' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' + cp /usr/local/pulsar/conf/standalone.conf /tmp/tidb_cdc_test/autorandom/pulsar_standalone.conf + pulsar_port=6650 + '[' normal == mtls ']' + '[' normal == oauth ']' + echo 'no cluster type specified, using default configuration.' no cluster type specified, using default configuration. ++ date + echo '[Fri Apr 4 15:39:00 CST 2025] <<<<<< START pulsar cluster in normal mode in autorandom case >>>>>>' [Fri Apr 4 15:39:00 CST 2025] <<<<<< START pulsar cluster in normal mode in autorandom case >>>>>> + echo 'Waiting for pulsar port to be ready...' Waiting for pulsar port to be ready... + i=0 + nc -z localhost 6650 + /usr/local/pulsar/bin/pulsar standalone --config /tmp/tidb_cdc_test/autorandom/pulsar_standalone.conf -nfw --metadata-dir /tmp/tidb_cdc_test/autorandom/pulsar-metadata --bookkeeper-dir /tmp/tidb_cdc_test/autorandom/pulsar-bookie + i=1 + '[' 1 -gt 20 ']' + sleep 2 table test.t1 exists + sleep 5 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/changefeed_auto_stop/run.sh using Sink-Type: pulsar... <<================= The 1 times to try to start tidb cluster... + nc -z localhost 6650 + i=2 + '[' 2 -gt 20 ']' + sleep 2 + nc -z localhost 6650 + echo 'Waiting for pulsar namespace to be ready...' Waiting for pulsar namespace to be ready... + i=0 + /usr/local/pulsar/bin/pulsar-admin namespaces list public start tidb cluster in /tmp/tidb_cdc_test/changefeed_auto_stop Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... + kill_tikv ++ ps aux ++ grep tikv-server ++ grep /tmp/tidb_cdc_test/synced_status_with_redo + info='jenkins 14142 25.2 0.6 5134820 2498600 ? Sl 15:38 0:05 tikv-server --pd 127.0.0.1:2379 -A 127.0.0.1:20160 --status-addr 127.0.0.1:20181 --log-file /tmp/tidb_cdc_test/synced_status_with_redo/tikv1.log --log-level debug -C /tmp/tidb_cdc_test/synced_status_with_redo/tikv-config.toml -s /tmp/tidb_cdc_test/synced_status_with_redo/tikv1 jenkins 14143 25.0 0.6 5141480 2506908 ? Sl 15:38 0:05 tikv-server --pd 127.0.0.1:2379 -A 127.0.0.1:20161 --status-addr 127.0.0.1:20182 --log-file /tmp/tidb_cdc_test/synced_status_with_redo/tikv2.log --log-level debug -C /tmp/tidb_cdc_test/synced_status_with_redo/tikv-config.toml -s /tmp/tidb_cdc_test/synced_status_with_redo/tikv2 jenkins 14144 33.7 0.6 5175272 2552952 ? Sl 15:38 0:06 tikv-server --pd 127.0.0.1:2379 -A 127.0.0.1:20162 --status-addr 127.0.0.1:20183 --log-file /tmp/tidb_cdc_test/synced_status_with_redo/tikv3.log --log-level debug -C /tmp/tidb_cdc_test/synced_status_with_redo/tikv-config.toml -s /tmp/tidb_cdc_test/synced_status_with_redo/tikv3 jenkins 14155 32.6 0.6 5158372 2539284 ? Sl 15:38 0:06 tikv-server --pd 127.0.0.1:2479 -A 127.0.0.1:21160 --status-addr 127.0.0.1:21180 --log-file /tmp/tidb_cdc_test/synced_status_with_redo/tikv_down.log --log-level debug -C /tmp/tidb_cdc_test/synced_status_with_redo/tikv-config.toml -s /tmp/tidb_cdc_test/synced_status_with_redo/tikv_down' ++ ps aux ++ grep tikv-server ++ grep /tmp/tidb_cdc_test/synced_status_with_redo ++ awk '{print $2}' ++ xargs kill -9 ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 243 100 243 0 0 2858 0 --:--:-- --:--:-- --:--:-- 2858 + synced_status='{"synced":false,"sink_checkpoint_ts":"2025-04-04 15:39:03.932","puller_resolved_ts":"2025-04-04 15:38:58.032","last_synced_ts":"2025-04-04 15:38:58.082","now_ts":"2025-04-04 15:39:05.000","info":"The data syncing is not finished, please wait"}' ++ echo '{"synced":false,"sink_checkpoint_ts":"2025-04-04' '15:39:03.932","puller_resolved_ts":"2025-04-04' '15:38:58.032","last_synced_ts":"2025-04-04' '15:38:58.082","now_ts":"2025-04-04' '15:39:05.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq .synced + status=false + '[' false '!=' false ']' ++ echo '{"synced":false,"sink_checkpoint_ts":"2025-04-04' '15:39:03.932","puller_resolved_ts":"2025-04-04' '15:38:58.032","last_synced_ts":"2025-04-04' '15:38:58.082","now_ts":"2025-04-04' '15:39:05.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq -r .info + info='The data syncing is not finished, please wait' + target_message='The data syncing is not finished, please wait' + '[' 'The data syncing is not finished, please wait' '!=' 'The data syncing is not finished, please wait' ']' + sleep 130 public/default ++ date + echo '[Fri Apr 4 15:39:06 CST 2025] <<<<<< pulsar is ready >>>>>>' [Fri Apr 4 15:39:06 CST 2025] <<<<<< pulsar is ready >>>>>> Create changefeed successfully! ID: 2270afc9-6d05-447e-97ab-48ece8ae9b7d Info: {"upstream_id":7489359227209379325,"namespace":"default","id":"2270afc9-6d05-447e-97ab-48ece8ae9b7d","sink_uri":"pulsar://127.0.0.1:6650/ticdc-autorandom-test-21261?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2025-04-04T15:39:07.34382332+08:00","start_ts":457114215167557634,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"pulsar_config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-10-ga306f4894","resolved_ts":457114215167557634,"checkpoint_ts":457114215167557634,"checkpoint_time":"2025-04-04 15:39:06.678"} [Fri Apr 4 15:39:07 CST 2025] <<<<<< START Pulsar consumer in autorandom case >>>>>> Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table autorandom_test.table_a not exists for 1-th check, retry later Starting Upstream TiDB... table autorandom_test.table_a exists Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 1-th time, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489359299229824862 TiDB Cluster ID. tikv_gc_leader_uuid 657fef531e4000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-2bnf-6v9zm, pid:12135, start at 2025-04-04 15:39:10.86990888 +0800 CST m=+1.541351340 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:41:10.877 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:39:10.841 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:29:10.841 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489359299229824862 TiDB Cluster ID. tikv_gc_leader_uuid 657fef531e4000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-2bnf-6v9zm, pid:12135, start at 2025-04-04 15:39:10.86990888 +0800 CST m=+1.541351340 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:41:10.877 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:39:10.841 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:29:10.841 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489359295853504241 TiDB Cluster ID. tikv_gc_leader_uuid 657fef532a4000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-2bnf-6v9zm, pid:12203, start at 2025-04-04 15:39:10.907227419 +0800 CST m=+1.514091919 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:41:10.916 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:39:10.889 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:29:10.889 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/changefeed_auto_stop/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/changefeed_auto_stop/tiflash/log/error.log arg matches is ArgMatches { args: {"memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/changefeed_auto_stop/tiflash/db/proxy"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/changefeed_auto_stop/tiflash/log/proxy.log"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/changefeed_auto_stop/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } check diff successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 4 15:39:13 CST 2025] <<<<<< run test case autorandom success! >>>>>> ***************** properties ***************** "updateproportion"="0" "mysql.user"="root" "insertproportion"="0" "dotransactions"="false" "requestdistribution"="uniform" "readallfields"="true" "mysql.port"="4000" "recordcount"="20" "readproportion"="0" "mysql.host"="127.0.0.1" "mysql.db"="changefeed_auto_stop_1" "operationcount"="0" "workload"="core" "threadcount"="4" "scanproportion"="0" ********************************************** Run finished, takes 9.75382ms INSERT - Takes(s): 0.0, Count: 20, OPS: 3545.5, Avg(us): 1840, Min(us): 1109, Max(us): 4038, 95th(us): 5000, 99th(us): 5000 ***************** properties ***************** "readallfields"="true" "workload"="core" "mysql.port"="4000" "requestdistribution"="uniform" "mysql.db"="changefeed_auto_stop_2" "insertproportion"="0" "recordcount"="20" "mysql.user"="root" "dotransactions"="false" "operationcount"="0" "readproportion"="0" "threadcount"="4" "scanproportion"="0" "updateproportion"="0" "mysql.host"="127.0.0.1" ********************************************** Run finished, takes 10.878991ms INSERT - Takes(s): 0.0, Count: 20, OPS: 3272.1, Avg(us): 2064, Min(us): 1008, Max(us): 4698, 95th(us): 5000, 99th(us): 5000 ***************** properties ***************** "operationcount"="0" "dotransactions"="false" "scanproportion"="0" "insertproportion"="0" "mysql.host"="127.0.0.1" "mysql.user"="root" "threadcount"="4" "mysql.db"="changefeed_auto_stop_3" "recordcount"="20" "readallfields"="true" "requestdistribution"="uniform" "workload"="core" "readproportion"="0" "mysql.port"="4000" "updateproportion"="0" ********************************************** Run finished, takes 8.617526ms INSERT - Takes(s): 0.0, Count: 20, OPS: 4295.2, Avg(us): 1579, Min(us): 865, Max(us): 3898, 95th(us): 4000, 99th(us): 4000 ***************** properties ***************** "updateproportion"="0" "scanproportion"="0" "operationcount"="0" "mysql.db"="changefeed_auto_stop_4" "mysql.port"="4000" "readproportion"="0" "workload"="core" "insertproportion"="0" "mysql.host"="127.0.0.1" "recordcount"="20" "dotransactions"="false" "mysql.user"="root" "threadcount"="4" "readallfields"="true" "requestdistribution"="uniform" ********************************************** Run finished, takes 10.599364ms INSERT - Takes(s): 0.0, Count: 19, OPS: 3014.8, Avg(us): 1756, Min(us): 998, Max(us): 4274, 95th(us): 5000, 99th(us): 5000 [Fri Apr 4 15:39:18 CST 2025] <<<<<< START cdc server in changefeed_auto_stop case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_auto_stop.1371313715.out server --log-file /tmp/tidb_cdc_test/changefeed_auto_stop/cdc1.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_auto_stop/cdc_data1 --cluster-id default --addr 127.0.0.1:8301 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8301 failed: Connection refused * Failed to connect to 127.0.0.1 port 8301: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8301 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 07:39:21 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [817 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/99640aa1-43d6-4048-a163-939939ec56d9 {"id":"99640aa1-43d6-4048-a163-939939ec56d9","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752358} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffbd327930 99640aa1-43d6-4048-a163-939939ec56d9 /tidb/cdc/default/default/upstream/7489359299229824862 {"id":7489359299229824862,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/99640aa1-43d6-4048-a163-939939ec56d9 {"id":"99640aa1-43d6-4048-a163-939939ec56d9","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752358} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffbd327930 99640aa1-43d6-4048-a163-939939ec56d9 /tidb/cdc/default/default/upstream/7489359299229824862 {"id":7489359299229824862,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/99640aa1-43d6-4048-a163-939939ec56d9 {"id":"99640aa1-43d6-4048-a163-939939ec56d9","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752358} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffbd327930 99640aa1-43d6-4048-a163-939939ec56d9 /tidb/cdc/default/default/upstream/7489359299229824862 {"id":7489359299229824862,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Fri Apr 4 15:39:21 CST 2025] <<<<<< START cdc server in changefeed_auto_stop case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/processor/pipeline/ProcessorSyncResolvedError=1*return(true);github.com/pingcap/tiflow/cdc/processor/ProcessorUpdatePositionDelaying=sleep(1000)' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_auto_stop.1377113773.out server --log-file /tmp/tidb_cdc_test/changefeed_auto_stop/cdc2.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_auto_stop/cdc_data2 --cluster-id default --addr 127.0.0.1:8302 --pd http://127.0.0.1:2379 + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8302 failed: Connection refused * Failed to connect to 127.0.0.1 port 8302: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8302 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8302 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 07:39:24 GMT < Content-Length: 1275 < Content-Type: text/plain; charset=utf-8 < { [1275 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/99640aa1-43d6-4048-a163-939939ec56d9 {"id":"99640aa1-43d6-4048-a163-939939ec56d9","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752358} /tidb/cdc/default/__cdc_meta__/capture/e1d1cd71-c0eb-4874-a86f-b2e2abb19f7d {"id":"e1d1cd71-c0eb-4874-a86f-b2e2abb19f7d","address":"127.0.0.1:8302","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752361} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffbd327930 99640aa1-43d6-4048-a163-939939ec56d9 /tidb/cdc/default/__cdc_meta__/owner/223195ffbd327966 e1d1cd71-c0eb-4874-a86f-b2e2abb19f7d /tidb/cdc/default/default/upstream/7489359299229824862 {"id":7489359299229824862,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/99640aa1-43d6-4048-a163-939939ec56d9 {"id":"99640aa1-43d6-4048-a163-939939ec56d9","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752358} /tidb/cdc/default/__cdc_meta__/capture/e1d1cd71-c0eb-4874-a86f-b2e2abb19f7d {"id":"e1d1cd71-c0eb-4874-a86f-b2e2abb19f7d","address":"127.0.0.1:8302","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752361} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffbd327930 99640aa1-43d6-4048-a163-939939ec56d9 /tidb/cdc/default/__cdc_meta__/owner/223195ffbd327966 e1d1cd71-c0eb-4874-a86f-b2e2abb19f7d /tidb/cdc/default/default/upstream/7489359299229824862 {"id":7489359299229824862,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/99640aa1-43d6-4048-a163-939939ec56d9 {"id":"99640aa1-43d6-4048-a163-939939ec56d9","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752358} /tidb/cdc/default/__cdc_meta__/capture/e1d1cd71-c0eb-4874-a86f-b2e2abb19f7d {"id":"e1d1cd71-c0eb-4874-a86f-b2e2abb19f7d","address":"127.0.0.1:8302","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752361} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffbd327930 99640aa1-43d6-4048-a163-939939ec56d9 /tidb/cdc/default/__cdc_meta__/owner/223195ffbd327966 e1d1cd71-c0eb-4874-a86f-b2e2abb19f7d /tidb/cdc/default/default/upstream/7489359299229824862 {"id":7489359299229824862,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x ++ date + echo '[Fri Apr 4 15:39:24 CST 2025] <<<<<< START pulsar cluster in changefeed_auto_stop case >>>>>>' [Fri Apr 4 15:39:24 CST 2025] <<<<<< START pulsar cluster in changefeed_auto_stop case >>>>>> + workdir=/tmp/tidb_cdc_test/changefeed_auto_stop + cluster_type=normal + cd /tmp/tidb_cdc_test/changefeed_auto_stop + DEFAULT_PULSAR_HOME=/usr/local/pulsar + pulsar_dir=/usr/local/pulsar ++ cat + mtls_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderTls brokerClientTlsEnabled=true brokerClientTrustCertsFilePath=/tmp/tidb_cdc_test/changefeed_auto_stop/ca.cert.pem brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls brokerClientAuthenticationParameters={"tlsCertFile":"/tmp/tidb_cdc_test/changefeed_auto_stop/broker_client.cert.pem","tlsKeyFile":"/tmp/tidb_cdc_test/changefeed_auto_stop/broker_client.key-pk8.pem"} brokerServicePortTls=6651 webServicePortTls=8443 tlsTrustCertsFilePath=/tmp/tidb_cdc_test/changefeed_auto_stop/ca.cert.pem tlsCertificateFilePath=/tmp/tidb_cdc_test/changefeed_auto_stop/server.cert.pem tlsKeyFilePath=/tmp/tidb_cdc_test/changefeed_auto_stop/server.key-pk8.pem tlsRequireTrustedClientCertOnConnect=true tlsAllowInsecureConnection=false tlsCertRefreshCheckDurationSec=300' ++ cat + normal_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' ++ cat + mtls_client_conf=' webServiceUrl=https://localhost:8443/ brokerServiceUrl=pulsar+ssl://localhost:6651/ authPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls authParams=tlsCertFile:/tmp/tidb_cdc_test/changefeed_auto_stop/broker_client.cert.pem,tlsKeyFile:/tmp/tidb_cdc_test/changefeed_auto_stop/broker_client.key-pk8.pem tlsTrustCertsFilePath=/tmp/tidb_cdc_test/changefeed_auto_stop/ca.cert.pem' ++ cat + oauth_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/ authPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 authParams={"privateKey":"/tmp/tidb_cdc_test/changefeed_auto_stop/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"}' ++ cat + oauth_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderToken brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 brokerClientAuthenticationParameters={"privateKey":"file:///tmp/tidb_cdc_test/changefeed_auto_stop/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"} tokenSecretKey=data:;base64,U0poWDM2X0thcFlTeWJCdEpxMzVseFhfQnJyNExSVVJTa203UW1YSkdteThwVUZXOUVJT2NWUVBzeWt6OS1qag==' ++ cat + credential_json=' { "client_id":"1234", "client_secret":"e0KVlA2EiBfjoN13olyZd2kv1KL", "audience":"cdc-api-uri", "issuer_url":"http://localhost:9096", "type": "client_credentials" }' ++ cat + cert_server_conf='[ req ] default_bits = 2048 prompt = no default_md = sha256 distinguished_name = dn [ v3_ext ] authorityKeyIdentifier=keyid,issuer:always basicConstraints=CA:FALSE keyUsage=critical, digitalSignature, keyEncipherment extendedKeyUsage=serverAuth subjectAltName=@alt_names [ dn ] CN = server [ alt_names ] DNS.1 = localhost IP.1 = 127.0.0.1' + echo ' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' + cp /usr/local/pulsar/conf/standalone.conf /tmp/tidb_cdc_test/changefeed_auto_stop/pulsar_standalone.conf + pulsar_port=6650 + '[' normal == mtls ']' + '[' normal == oauth ']' + echo 'no cluster type specified, using default configuration.' no cluster type specified, using default configuration. ++ date + echo '[Fri Apr 4 15:39:24 CST 2025] <<<<<< START pulsar cluster in normal mode in changefeed_auto_stop case >>>>>>' [Fri Apr 4 15:39:24 CST 2025] <<<<<< START pulsar cluster in normal mode in changefeed_auto_stop case >>>>>> + echo 'Waiting for pulsar port to be ready...' Waiting for pulsar port to be ready... + i=0 + nc -z localhost 6650 + /usr/local/pulsar/bin/pulsar standalone --config /tmp/tidb_cdc_test/changefeed_auto_stop/pulsar_standalone.conf -nfw --metadata-dir /tmp/tidb_cdc_test/changefeed_auto_stop/pulsar-metadata --bookkeeper-dir /tmp/tidb_cdc_test/changefeed_auto_stop/pulsar-bookie + i=1 + '[' 1 -gt 20 ']' + sleep 2 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/generate_column/run.sh using Sink-Type: pulsar... <<================= The 1 times to try to start tidb cluster... + nc -z localhost 6650 + i=2 + '[' 2 -gt 20 ']' + sleep 2 start tidb cluster in /tmp/tidb_cdc_test/generate_column Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... + nc -z localhost 6650 + echo 'Waiting for pulsar namespace to be ready...' Waiting for pulsar namespace to be ready... + i=0 + /usr/local/pulsar/bin/pulsar-admin namespaces list public Verifying downstream PD is started... public/default ++ date + echo '[Fri Apr 4 15:39:30 CST 2025] <<<<<< pulsar is ready >>>>>>' [Fri Apr 4 15:39:30 CST 2025] <<<<<< pulsar is ready >>>>>> Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release [Fri Apr 4 15:39:31 CST 2025] <<<<<< START Pulsar consumer in changefeed_auto_stop case >>>>>> check_changefeed_state http://127.0.0.1:2379 029a8104-ce0f-4e1b-bd74-ea975674a0c6 normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=029a8104-ce0f-4e1b-bd74-ea975674a0c6 + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c 029a8104-ce0f-4e1b-bd74-ea975674a0c6 -s + info='{ "upstream_id": 7489359299229824862, "namespace": "default", "id": "029a8104-ce0f-4e1b-bd74-ea975674a0c6", "state": "normal", "checkpoint_tso": 457114217032450049, "checkpoint_time": "2025-04-04 15:39:13.792", "error": null }' + echo '{ "upstream_id": 7489359299229824862, "namespace": "default", "id": "029a8104-ce0f-4e1b-bd74-ea975674a0c6", "state": "normal", "checkpoint_tso": 457114217032450049, "checkpoint_time": "2025-04-04 15:39:13.792", "error": null }' { "upstream_id": 7489359299229824862, "namespace": "default", "id": "029a8104-ce0f-4e1b-bd74-ea975674a0c6", "state": "normal", "checkpoint_tso": 457114217032450049, "checkpoint_time": "2025-04-04 15:39:13.792", "error": null } ++ echo '{' '"upstream_id":' 7489359299229824862, '"namespace":' '"default",' '"id":' '"029a8104-ce0f-4e1b-bd74-ea975674a0c6",' '"state":' '"normal",' '"checkpoint_tso":' 457114217032450049, '"checkpoint_time":' '"2025-04-04' '15:39:13.792",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7489359299229824862, '"namespace":' '"default",' '"id":' '"029a8104-ce0f-4e1b-bd74-ea975674a0c6",' '"state":' '"normal",' '"checkpoint_tso":' 457114217032450049, '"checkpoint_time":' '"2025-04-04' '15:39:13.792",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully table changefeed_auto_stop_1.usertable not exists for 1-th check, retry later + '[' pulsar == mysql ']' + stop_tidb_cluster Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table changefeed_auto_stop_1.usertable exists table changefeed_auto_stop_2.usertable not exists for 1-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489359394420960011 TiDB Cluster ID. tikv_gc_leader_uuid 657fef6a6c00007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-6nlr-p88d4, pid:35241, start at 2025-04-04 15:39:34.714488984 +0800 CST m=+1.990938104 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:41:34.722 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:39:34.704 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:29:34.704 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table changefeed_auto_stop_2.usertable exists table changefeed_auto_stop_3.usertable not exists for 1-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489359394420960011 TiDB Cluster ID. tikv_gc_leader_uuid 657fef6a6c00007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-6nlr-p88d4, pid:35241, start at 2025-04-04 15:39:34.714488984 +0800 CST m=+1.990938104 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:41:34.722 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:39:34.704 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:29:34.704 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489359397452585682 TiDB Cluster ID. tikv_gc_leader_uuid 657fef6a874000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-6nlr-p88d4, pid:35310, start at 2025-04-04 15:39:34.852058456 +0800 CST m=+2.054999863 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:41:34.864 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:39:34.863 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:29:34.863 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/generate_column/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/generate_column/tiflash/log/error.log arg matches is ArgMatches { args: {"memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/generate_column/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/generate_column/tiflash-proxy.toml"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/generate_column/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table changefeed_auto_stop_3.usertable exists table changefeed_auto_stop_4.usertable not exists for 1-th check, retry later + stop_tidb_cluster + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.generate_column.cli.36699.out cli tso query --pd=http://127.0.0.1:2379 table changefeed_auto_stop_4.usertable exists check diff failed 1-th time, retry later + set +x + tso='457114223772696577 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 457114223772696577 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Fri Apr 4 15:39:40 CST 2025] <<<<<< START cdc server in generate_column case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.generate_column.3674136743.out server --log-file /tmp/tidb_cdc_test/generate_column/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/generate_column/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff successfully wait process cdc.test exit for 1-th time... \033[0;36m<<< Run all test success >>>\033[0m wait process cdc.test exit for 2-th time... [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-3484/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 4 15:39:43 CST 2025] <<<<<< run test case changefeed_auto_stop success! >>>>>> [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 07:39:44 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [817 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7e9e0af1-b574-4508-bc2c-4f6b9d2297e9 {"id":"7e9e0af1-b574-4508-bc2c-4f6b9d2297e9","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752381} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffbd8bc9ea 7e9e0af1-b574-4508-bc2c-4f6b9d2297e9 /tidb/cdc/default/default/upstream/7489359394420960011 {"id":7489359394420960011,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7e9e0af1-b574-4508-bc2c-4f6b9d2297e9 {"id":"7e9e0af1-b574-4508-bc2c-4f6b9d2297e9","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752381} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffbd8bc9ea 7e9e0af1-b574-4508-bc2c-4f6b9d2297e9 /tidb/cdc/default/default/upstream/7489359394420960011 {"id":7489359394420960011,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7e9e0af1-b574-4508-bc2c-4f6b9d2297e9 {"id":"7e9e0af1-b574-4508-bc2c-4f6b9d2297e9","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752381} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffbd8bc9ea 7e9e0af1-b574-4508-bc2c-4f6b9d2297e9 /tidb/cdc/default/default/upstream/7489359394420960011 {"id":7489359394420960011,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x ++ date + echo '[Fri Apr 4 15:39:44 CST 2025] <<<<<< START pulsar cluster in generate_column case >>>>>>' [Fri Apr 4 15:39:44 CST 2025] <<<<<< START pulsar cluster in generate_column case >>>>>> + workdir=/tmp/tidb_cdc_test/generate_column + cluster_type=normal + cd /tmp/tidb_cdc_test/generate_column + DEFAULT_PULSAR_HOME=/usr/local/pulsar + pulsar_dir=/usr/local/pulsar ++ cat + mtls_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderTls brokerClientTlsEnabled=true brokerClientTrustCertsFilePath=/tmp/tidb_cdc_test/generate_column/ca.cert.pem brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls brokerClientAuthenticationParameters={"tlsCertFile":"/tmp/tidb_cdc_test/generate_column/broker_client.cert.pem","tlsKeyFile":"/tmp/tidb_cdc_test/generate_column/broker_client.key-pk8.pem"} brokerServicePortTls=6651 webServicePortTls=8443 tlsTrustCertsFilePath=/tmp/tidb_cdc_test/generate_column/ca.cert.pem tlsCertificateFilePath=/tmp/tidb_cdc_test/generate_column/server.cert.pem tlsKeyFilePath=/tmp/tidb_cdc_test/generate_column/server.key-pk8.pem tlsRequireTrustedClientCertOnConnect=true tlsAllowInsecureConnection=false tlsCertRefreshCheckDurationSec=300' ++ cat + normal_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' ++ cat + mtls_client_conf=' webServiceUrl=https://localhost:8443/ brokerServiceUrl=pulsar+ssl://localhost:6651/ authPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls authParams=tlsCertFile:/tmp/tidb_cdc_test/generate_column/broker_client.cert.pem,tlsKeyFile:/tmp/tidb_cdc_test/generate_column/broker_client.key-pk8.pem tlsTrustCertsFilePath=/tmp/tidb_cdc_test/generate_column/ca.cert.pem' ++ cat + oauth_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/ authPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 authParams={"privateKey":"/tmp/tidb_cdc_test/generate_column/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"}' ++ cat + oauth_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderToken brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 brokerClientAuthenticationParameters={"privateKey":"file:///tmp/tidb_cdc_test/generate_column/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"} tokenSecretKey=data:;base64,U0poWDM2X0thcFlTeWJCdEpxMzVseFhfQnJyNExSVVJTa203UW1YSkdteThwVUZXOUVJT2NWUVBzeWt6OS1qag==' ++ cat + credential_json=' { "client_id":"1234", "client_secret":"e0KVlA2EiBfjoN13olyZd2kv1KL", "audience":"cdc-api-uri", "issuer_url":"http://localhost:9096", "type": "client_credentials" }' ++ cat + cert_server_conf='[ req ] default_bits = 2048 prompt = no default_md = sha256 distinguished_name = dn [ v3_ext ] authorityKeyIdentifier=keyid,issuer:always basicConstraints=CA:FALSE keyUsage=critical, digitalSignature, keyEncipherment extendedKeyUsage=serverAuth subjectAltName=@alt_names [ dn ] CN = server [ alt_names ] DNS.1 = localhost IP.1 = 127.0.0.1' + echo ' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' + cp /usr/local/pulsar/conf/standalone.conf /tmp/tidb_cdc_test/generate_column/pulsar_standalone.conf + pulsar_port=6650 + '[' normal == mtls ']' + '[' normal == oauth ']' + echo 'no cluster type specified, using default configuration.' no cluster type specified, using default configuration. ++ date + echo '[Fri Apr 4 15:39:44 CST 2025] <<<<<< START pulsar cluster in normal mode in generate_column case >>>>>>' [Fri Apr 4 15:39:44 CST 2025] <<<<<< START pulsar cluster in normal mode in generate_column case >>>>>> + echo 'Waiting for pulsar port to be ready...' Waiting for pulsar port to be ready... + i=0 + nc -z localhost 6650 + /usr/local/pulsar/bin/pulsar standalone --config /tmp/tidb_cdc_test/generate_column/pulsar_standalone.conf -nfw --metadata-dir /tmp/tidb_cdc_test/generate_column/pulsar-metadata --bookkeeper-dir /tmp/tidb_cdc_test/generate_column/pulsar-bookie + i=1 + '[' 1 -gt 20 ']' + sleep 2 + nc -z localhost 6650 + i=2 + '[' 2 -gt 20 ']' + sleep 2 + nc -z localhost 6650 + echo 'Waiting for pulsar namespace to be ready...' Waiting for pulsar namespace to be ready... + i=0 + /usr/local/pulsar/bin/pulsar-admin namespaces list public public/default ++ date + echo '[Fri Apr 4 15:39:50 CST 2025] <<<<<< pulsar is ready >>>>>>' [Fri Apr 4 15:39:50 CST 2025] <<<<<< pulsar is ready >>>>>> + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.generate_column.cli.37202.out cli changefeed create --start-ts=457114223772696577 '--sink-uri=pulsar://127.0.0.1:6650/ticdc-generate-column-test-25233?protocol=canal-json&enable-tidb-extension=true' Create changefeed successfully! ID: 7f4ed561-2f2c-4579-83fd-71ec31072843 Info: {"upstream_id":7489359394420960011,"namespace":"default","id":"7f4ed561-2f2c-4579-83fd-71ec31072843","sink_uri":"pulsar://127.0.0.1:6650/ticdc-generate-column-test-25233?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2025-04-04T15:39:51.368520558+08:00","start_ts":457114223772696577,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"pulsar_config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-10-ga306f4894","resolved_ts":457114223772696577,"checkpoint_ts":457114223772696577,"checkpoint_time":"2025-04-04 15:39:39.504"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... + set +x [Fri Apr 4 15:39:52 CST 2025] <<<<<< START Pulsar consumer in generate_column case >>>>>> table generate_column.t1 not exists for 1-th check, retry later \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-3484/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } table generate_column.t1 exists check diff failed 1-th time, retry later check diff successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri Apr 4 15:39:58 CST 2025] <<<<<< run test case generate_column success! >>>>>> \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-3484/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 221 100 221 0 0 7129 0 --:--:-- --:--:-- --:--:-- 7129 + synced_status='{"synced":true,"sink_checkpoint_ts":"2025-04-04 15:40:37.246","puller_resolved_ts":"2025-04-04 15:40:29.246","last_synced_ts":"2025-04-04 15:38:20.746","now_ts":"2025-04-04 15:40:37.000","info":"Data syncing is finished"}' ++ echo '{"synced":true,"sink_checkpoint_ts":"2025-04-04' '15:40:37.246","puller_resolved_ts":"2025-04-04' '15:40:29.246","last_synced_ts":"2025-04-04' '15:38:20.746","now_ts":"2025-04-04' '15:40:37.000","info":"Data' syncing is 'finished"}' ++ jq .synced + status=true + '[' true '!=' true ']' + kill_pd ++ ps aux ++ grep pd-server ++ grep /tmp/tidb_cdc_test/synced_status + info='jenkins 17459 9.4 0.0 15707372 145184 ? Sl 15:38 0:14 pd-server --advertise-client-urls http://127.0.0.1:2379 --client-urls http://0.0.0.0:2379 --advertise-peer-urls http://127.0.0.1:2380 --peer-urls http://0.0.0.0:2380 --config /tmp/tidb_cdc_test/synced_status/pd-config.toml --log-file /tmp/tidb_cdc_test/synced_status/pd1.log --data-dir /tmp/tidb_cdc_test/synced_status/pd1 --name=pd1 --initial-cluster=pd1=http://127.0.0.1:2380 jenkins 17526 6.7 0.0 15705836 142736 ? Sl 15:38 0:10 pd-server --advertise-client-urls http://127.0.0.1:2479 --client-urls http://0.0.0.0:2479 --advertise-peer-urls http://127.0.0.1:2480 --peer-urls http://0.0.0.0:2480 --config /tmp/tidb_cdc_test/synced_status/pd-config.toml --log-file /tmp/tidb_cdc_test/synced_status/down_pd.log --data-dir /tmp/tidb_cdc_test/synced_status/down_pd' ++ ps aux ++ grep pd-server ++ grep /tmp/tidb_cdc_test/synced_status ++ awk '{print $2}' ++ xargs kill -9 + sleep 20 {"level":"warn","ts":"2025-04-04T15:40:44.121743+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc002084e00/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":"2025-04-04T15:40:44.121795+0800","logger":"etcd-client","caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":"2025-04-04T15:40:44.123452+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001249dc0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":"2025-04-04T15:40:44.123521+0800","logger":"etcd-client","caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":"2025-04-04T15:40:44.131998+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc002084e00/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2025-04-04T15:40:44.132816+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001249dc0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2025-04-04T15:40:44.188059+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000985a40/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} {"level":"info","ts":"2025-04-04T15:40:44.188137+0800","logger":"etcd-client","caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":"2025-04-04T15:40:44.199385+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000985a40/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} {"level":"warn","ts":1743752445.6944273,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00544c1c0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":1743752445.6945415,"caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":1743752445.6963663,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc003aaddc0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":1743752445.6964629,"caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":1743752445.7405472,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00539f180/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} {"level":"info","ts":1743752445.7406063,"caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":"2025-04-04T15:40:50.133866+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc002084e00/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2025-04-04T15:40:50.13489+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001249dc0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2025-04-04T15:40:50.200414+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000985a40/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} {"level":"warn","ts":"2025-04-04T15:40:56.13436+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc002084e00/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2025-04-04T15:40:56.136126+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001249dc0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2025-04-04T15:40:56.202136+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000985a40/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0{"level":"warn","ts":1743752458.778192,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00544c1c0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":1743752458.8300781,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00539f180/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} 0 0 0 0 0 0 0 0 --:--:-- 0:00:01 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:02 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:03 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:04 --:--:-- 0{"level":"warn","ts":"2025-04-04T15:41:02.135733+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc002084e00/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2025-04-04T15:41:02.137728+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001249dc0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2025-04-04T15:41:02.203193+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000985a40/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} 0 0 0 0 0 0 0 0 --:--:-- 0:00:05 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:06 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:07 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:08 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:09 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:10 --:--:-- 0{"level":"warn","ts":"2025-04-04T15:41:08.137545+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc002084e00/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2025-04-04T15:41:08.138663+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001249dc0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2025-04-04T15:41:08.204174+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000985a40/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} 0 0 0 0 0 0 0 0 --:--:-- 0:00:11 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:12 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:13 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:14 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:15 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:16 --:--:-- 0{"level":"warn","ts":"2025-04-04T15:41:14.13869+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc002084e00/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2025-04-04T15:41:14.140283+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001249dc0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2025-04-04T15:41:14.206119+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000985a40/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} 0 0 0 0 0 0 0 0 --:--:-- 0:00:17 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:18 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:19 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:20 --:--:-- 0{"level":"warn","ts":"2025-04-04T15:41:19.122464+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc002084e00/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":"2025-04-04T15:41:19.122514+0800","logger":"etcd-client","caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":"2025-04-04T15:41:19.124014+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001249dc0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":"2025-04-04T15:41:19.124059+0800","logger":"etcd-client","caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} 0 0 0 0 0 0 0 0 --:--:-- 0:00:21 --:--:-- 0{"level":"warn","ts":"2025-04-04T15:41:19.189122+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000985a40/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} {"level":"info","ts":"2025-04-04T15:41:19.189182+0800","logger":"etcd-client","caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} 0 0 0 0 0 0 0 0 --:--:-- 0:00:22 --:--:-- 0{"level":"warn","ts":"2025-04-04T15:41:20.139684+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc002084e00/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2025-04-04T15:41:20.141643+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001249dc0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2025-04-04T15:41:20.206741+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000985a40/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} {"level":"warn","ts":1743752480.6958437,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00544c1c0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":1743752480.6959083,"caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":1743752480.6980126,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc003aaddc0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":1743752480.6980634,"caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":1743752480.7423306,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00539f180/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} {"level":"info","ts":1743752480.7423859,"caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} 0 0 0 0 0 0 0 0 --:--:-- 0:00:23 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:24 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:25 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:26 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:27 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:28 --:--:-- 0{"level":"warn","ts":"2025-04-04T15:41:26.140841+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc002084e00/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2025-04-04T15:41:26.142897+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001249dc0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2025-04-04T15:41:26.208725+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000985a40/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 723 100 723 0 0 24100 0 --:--:-- --:--:-- --:--:-- 24100 + synced_status='{"synced":false,"sink_checkpoint_ts":"2025-04-04 15:39:04.932","puller_resolved_ts":"2025-04-04 15:39:04.932","last_synced_ts":"2025-04-04 15:38:58.082","now_ts":"2025-04-04 15:41:15.000","info":"Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' \u003e '\''Resolved-Ts'\'' \u003e '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait"}' ++ echo '{"synced":false,"sink_checkpoint_ts":"2025-04-04' '15:39:04.932","puller_resolved_ts":"2025-04-04' '15:39:04.932","last_synced_ts":"2025-04-04' '15:38:58.082","now_ts":"2025-04-04' '15:41:15.000","info":"Please' check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view ''\''TiKV-Details'\''' '\u003e' ''\''Resolved-Ts'\''' '\u003e' ''\''Max' Leader Resolved TS 'gap'\''' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please 'wait"}' ++ jq .synced + status=false + '[' false '!=' false ']' ++ jq -r .info ++ echo '{"synced":false,"sink_checkpoint_ts":"2025-04-04' '15:39:04.932","puller_resolved_ts":"2025-04-04' '15:39:04.932","last_synced_ts":"2025-04-04' '15:38:58.082","now_ts":"2025-04-04' '15:41:15.000","info":"Please' check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view ''\''TiKV-Details'\''' '\u003e' ''\''Resolved-Ts'\''' '\u003e' ''\''Max' Leader Resolved TS 'gap'\''' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please 'wait"}' + info='Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' + target_message='Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' + '[' 'Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' '!=' 'Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' ']' + cleanup_process cdc.test wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit + stop_tidb_cluster + run_case_with_unavailable_tidb conf/changefeed-redo.toml + rm -rf /tmp/tidb_cdc_test/synced_status_with_redo + mkdir -p /tmp/tidb_cdc_test/synced_status_with_redo + start_tidb_cluster --workdir /tmp/tidb_cdc_test/synced_status_with_redo shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory The 1 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory 0 0 0 0 0 0 0 0 --:--:-- 0:00:29 --:--:-- 0 100 135 100 135 0 0 4 0 0:00:33 0:00:30 0:00:03 28 100 135 100 135 0 0 4 0 0:00:33 0:00:30 0:00:03 35 + synced_status='{ "error_msg": "[CDC:ErrPDEtcdAPIError]etcd api call error: context deadline exceeded", "error_code": "CDC:ErrPDEtcdAPIError" }' ++ echo '{' '"error_msg":' '"[CDC:ErrPDEtcdAPIError]etcd' api call error: context deadline 'exceeded",' '"error_code":' '"CDC:ErrPDEtcdAPIError"' '}' ++ jq -r .error_code + error_code=CDC:ErrPDEtcdAPIError + cleanup_process cdc.test chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/synced_status_with_redo Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit + stop_tidb_cluster Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489359917602943691 TiDB Cluster ID. tikv_gc_leader_uuid 657fefe04a40004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-9bld-w6hq9, pid:17643, start at 2025-04-04 15:41:35.406889727 +0800 CST m=+1.468366568 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:43:35.414 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:41:35.401 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:31:35.401 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489359917602943691 TiDB Cluster ID. tikv_gc_leader_uuid 657fefe04a40004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-9bld-w6hq9, pid:17643, start at 2025-04-04 15:41:35.406889727 +0800 CST m=+1.468366568 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:43:35.414 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:41:35.401 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:31:35.401 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489359917502045045 TiDB Cluster ID. tikv_gc_leader_uuid 657fefe0618000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-9bld-w6hq9, pid:17708, start at 2025-04-04 15:41:35.528414792 +0800 CST m=+1.527025789 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:43:35.535 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:41:35.494 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:31:35.494 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/synced_status_with_redo/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/synced_status_with_redo/tiflash/log/error.log arg matches is ArgMatches { args: {"config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/synced_status_with_redo/tiflash-proxy.toml"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/synced_status_with_redo/tiflash/log/proxy.log"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/synced_status_with_redo/tiflash/db/proxy"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + run_case_with_unavailable_tikv conf/changefeed.toml + rm -rf /tmp/tidb_cdc_test/synced_status + mkdir -p /tmp/tidb_cdc_test/synced_status + start_tidb_cluster --workdir /tmp/tidb_cdc_test/synced_status shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory The 1 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory + cd /tmp/tidb_cdc_test/synced_status_with_redo ++ run_cdc_cli_tso_query 127.0.0.1 2379 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status_with_redo.cli.19161.out cli tso query --pd=http://127.0.0.1:2379 + set +x + tso='457114255006367745 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 457114255006367745 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + start_ts=457114255006367745 + run_cdc_server --workdir /tmp/tidb_cdc_test/synced_status_with_redo --binary cdc.test [Fri Apr 4 15:41:40 CST 2025] <<<<<< START cdc server in synced_status_with_redo case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status_with_redo.1919819200.out server --log-file /tmp/tidb_cdc_test/synced_status_with_redo/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/synced_status_with_redo/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/synced_status Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 07:41:43 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [817 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7c4dc29e-d400-487b-9217-cf8e19fd15c4 {"id":"7c4dc29e-d400-487b-9217-cf8e19fd15c4","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752500} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffbf6568d9 7c4dc29e-d400-487b-9217-cf8e19fd15c4 /tidb/cdc/default/default/upstream/7489359917602943691 {"id":7489359917602943691,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7c4dc29e-d400-487b-9217-cf8e19fd15c4 {"id":"7c4dc29e-d400-487b-9217-cf8e19fd15c4","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752500} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffbf6568d9 7c4dc29e-d400-487b-9217-cf8e19fd15c4 /tidb/cdc/default/default/upstream/7489359917602943691 {"id":7489359917602943691,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7c4dc29e-d400-487b-9217-cf8e19fd15c4 {"id":"7c4dc29e-d400-487b-9217-cf8e19fd15c4","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752500} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffbf6568d9 7c4dc29e-d400-487b-9217-cf8e19fd15c4 /tidb/cdc/default/default/upstream/7489359917602943691 {"id":7489359917602943691,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + config_path=conf/changefeed-redo.toml + SINK_URI='mysql://root@127.0.0.1:3306/?max-txn-row=1' + run_cdc_cli changefeed create --start-ts=457114255006367745 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/synced_status_with_redo/conf/changefeed-redo.toml + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status_with_redo.cli.19250.out cli changefeed create --start-ts=457114255006367745 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/synced_status_with_redo/conf/changefeed-redo.toml Create changefeed successfully! ID: test-1 Info: {"upstream_id":7489359917602943691,"namespace":"default","id":"test-1","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2025-04-04T15:41:43.621983308+08:00","start_ts":457114255006367745,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"eventual","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"storage":"file:///tmp/tidb_cdc_test/synced_status/redo","use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":120,"checkpoint_interval":20}},"state":"normal","creator_version":"v9.0.0-beta.1-10-ga306f4894","resolved_ts":457114255006367745,"checkpoint_ts":457114255006367745,"checkpoint_time":"2025-04-04 15:41:38.651"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + set +x + run_sql 'USE TEST;Create table t1(a int primary key, b int);insert into t1 values(1,2);insert into t1 values(2,3);' Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore + check_table_exists test.t1 127.0.0.1 3306 table test.t1 not exists for 1-th check, retry later Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489359971490943424 TiDB Cluster ID. tikv_gc_leader_uuid 657fefebcb00007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-qg8g-pl0pp, pid:21144, start at 2025-04-04 15:41:47.192968838 +0800 CST m=+1.740188276 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:43:47.200 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:41:47.180 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:31:47.180 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489359971490943424 TiDB Cluster ID. tikv_gc_leader_uuid 657fefebcb00007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-qg8g-pl0pp, pid:21144, start at 2025-04-04 15:41:47.192968838 +0800 CST m=+1.740188276 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:43:47.200 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:41:47.180 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:31:47.180 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489359969705567499 TiDB Cluster ID. tikv_gc_leader_uuid 657fefebd6c000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-qg8g-pl0pp, pid:21228, start at 2025-04-04 15:41:47.271845062 +0800 CST m=+1.742883587 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:43:47.282 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:41:47.278 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:31:47.278 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/synced_status/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/synced_status/tiflash/log/error.log arg matches is ArgMatches { args: {"pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash/log/proxy.log"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash/db/proxy"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table test.t1 exists + sleep 5 + cd /tmp/tidb_cdc_test/synced_status ++ run_cdc_cli_tso_query 127.0.0.1 2379 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.22687.out cli tso query --pd=http://127.0.0.1:2379 + set +x + tso='457114258028888076 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 457114258028888076 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + start_ts=457114258028888076 + run_cdc_server --workdir /tmp/tidb_cdc_test/synced_status --binary cdc.test [Fri Apr 4 15:41:51 CST 2025] <<<<<< START cdc server in synced_status case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.2271122713.out server --log-file /tmp/tidb_cdc_test/synced_status/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/synced_status/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + kill_tidb ++ ps aux ++ grep /tmp/tidb_cdc_test/synced_status_with_redo ++ grep tidb-server + info='jenkins 17634 4.6 0.0 3332068 198040 ? Sl 15:41 0:00 tidb-server -P 4000 -config /tmp/tidb_cdc_test/synced_status_with_redo/tidb-config-1743752493916828013.toml --store tikv --path 127.0.0.1:2379 --status=10080 --log-file /tmp/tidb_cdc_test/synced_status_with_redo/tidb.log jenkins 17643 9.7 0.0 3184860 214868 ? Sl 15:41 0:01 tidb-server -P 4001 -config /tmp/tidb_cdc_test/synced_status_with_redo/tidb-config-1743752493925435452.toml --store tikv --path 127.0.0.1:2379 --status=10081 --log-file /tmp/tidb_cdc_test/synced_status_with_redo/tidb_other.log jenkins 17708 11.0 0.0 3398792 237560 ? Sl 15:41 0:02 tidb-server -P 3306 -config /tmp/tidb_cdc_test/synced_status_with_redo/tidb-config-1743752493989632559.toml --store tikv --path 127.0.0.1:2479 --status=20080 --log-file /tmp/tidb_cdc_test/synced_status_with_redo/tidb_down.log' ++ ps aux ++ grep tidb-server ++ grep /tmp/tidb_cdc_test/synced_status_with_redo ++ awk '{print $2}' ++ xargs kill -9 ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 243 100 243 0 0 17357 0 --:--:-- --:--:-- --:--:-- 17357 + synced_status='{"synced":false,"sink_checkpoint_ts":"2025-04-04 15:41:50.951","puller_resolved_ts":"2025-04-04 15:41:45.051","last_synced_ts":"2025-04-04 15:41:45.552","now_ts":"2025-04-04 15:41:52.000","info":"The data syncing is not finished, please wait"}' ++ echo '{"synced":false,"sink_checkpoint_ts":"2025-04-04' '15:41:50.951","puller_resolved_ts":"2025-04-04' '15:41:45.051","last_synced_ts":"2025-04-04' '15:41:45.552","now_ts":"2025-04-04' '15:41:52.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq .synced + status=false + '[' false '!=' false ']' ++ echo '{"synced":false,"sink_checkpoint_ts":"2025-04-04' '15:41:50.951","puller_resolved_ts":"2025-04-04' '15:41:45.051","last_synced_ts":"2025-04-04' '15:41:45.552","now_ts":"2025-04-04' '15:41:52.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq -r .info + info='The data syncing is not finished, please wait' + target_message='The data syncing is not finished, please wait' + '[' 'The data syncing is not finished, please wait' '!=' 'The data syncing is not finished, please wait' ']' + sleep 130 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 07:41:54 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [817 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f40414d0-9d86-43cd-995f-a39ee76d19fc {"id":"f40414d0-9d86-43cd-995f-a39ee76d19fc","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752512} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffbf9657d8 f40414d0-9d86-43cd-995f-a39ee76d19fc /tidb/cdc/default/default/upstream/7489359971490943424 {"id":7489359971490943424,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f40414d0-9d86-43cd-995f-a39ee76d19fc {"id":"f40414d0-9d86-43cd-995f-a39ee76d19fc","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752512} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffbf9657d8 f40414d0-9d86-43cd-995f-a39ee76d19fc /tidb/cdc/default/default/upstream/7489359971490943424 {"id":7489359971490943424,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f40414d0-9d86-43cd-995f-a39ee76d19fc {"id":"f40414d0-9d86-43cd-995f-a39ee76d19fc","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752512} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffbf9657d8 f40414d0-9d86-43cd-995f-a39ee76d19fc /tidb/cdc/default/default/upstream/7489359971490943424 {"id":7489359971490943424,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + config_path=conf/changefeed.toml + SINK_URI='mysql://root@127.0.0.1:3306/?max-txn-row=1' + run_cdc_cli changefeed create --start-ts=457114258028888076 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/synced_status/conf/changefeed.toml + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.22769.out cli changefeed create --start-ts=457114258028888076 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/synced_status/conf/changefeed.toml Create changefeed successfully! ID: test-1 Info: {"upstream_id":7489359971490943424,"namespace":"default","id":"test-1","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2025-04-04T15:41:55.139927356+08:00","start_ts":457114258028888076,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":120,"checkpoint_interval":20}},"state":"normal","creator_version":"v9.0.0-beta.1-10-ga306f4894","resolved_ts":457114258028888076,"checkpoint_ts":457114258028888076,"checkpoint_time":"2025-04-04 15:41:50.181"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x + run_sql 'USE TEST;Create table t1(a int primary key, b int);insert into t1 values(1,2);insert into t1 values(2,3);' + check_table_exists test.t1 127.0.0.1 3306 table test.t1 not exists for 1-th check, retry later table test.t1 exists + sleep 5 + kill_tikv ++ ps aux ++ grep tikv-server ++ grep /tmp/tidb_cdc_test/synced_status + info='jenkins 20487 37.3 0.6 5168104 2553380 ? Sl 15:41 0:07 tikv-server --pd 127.0.0.1:2379 -A 127.0.0.1:20160 --status-addr 127.0.0.1:20181 --log-file /tmp/tidb_cdc_test/synced_status/tikv1.log --log-level debug -C /tmp/tidb_cdc_test/synced_status/tikv-config.toml -s /tmp/tidb_cdc_test/synced_status/tikv1 jenkins 20488 25.6 0.6 5118948 2477216 ? Sl 15:41 0:05 tikv-server --pd 127.0.0.1:2379 -A 127.0.0.1:20161 --status-addr 127.0.0.1:20182 --log-file /tmp/tidb_cdc_test/synced_status/tikv2.log --log-level debug -C /tmp/tidb_cdc_test/synced_status/tikv-config.toml -s /tmp/tidb_cdc_test/synced_status/tikv2 jenkins 20489 26.1 0.6 5118948 2477540 ? Sl 15:41 0:05 tikv-server --pd 127.0.0.1:2379 -A 127.0.0.1:20162 --status-addr 127.0.0.1:20183 --log-file /tmp/tidb_cdc_test/synced_status/tikv3.log --log-level debug -C /tmp/tidb_cdc_test/synced_status/tikv-config.toml -s /tmp/tidb_cdc_test/synced_status/tikv3 jenkins 20491 35.4 0.6 5162468 2539712 ? Sl 15:41 0:07 tikv-server --pd 127.0.0.1:2479 -A 127.0.0.1:21160 --status-addr 127.0.0.1:21180 --log-file /tmp/tidb_cdc_test/synced_status/tikv_down.log --log-level debug -C /tmp/tidb_cdc_test/synced_status/tikv-config.toml -s /tmp/tidb_cdc_test/synced_status/tikv_down' ++ ps aux ++ grep tikv-server ++ grep /tmp/tidb_cdc_test/synced_status ++ awk '{print $2}' ++ xargs kill -9 ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 243 100 243 0 0 6567 0 --:--:-- --:--:-- --:--:-- 6567 + synced_status='{"synced":false,"sink_checkpoint_ts":"2025-04-04 15:42:02.481","puller_resolved_ts":"2025-04-04 15:41:56.581","last_synced_ts":"2025-04-04 15:41:56.630","now_ts":"2025-04-04 15:42:03.000","info":"The data syncing is not finished, please wait"}' ++ echo '{"synced":false,"sink_checkpoint_ts":"2025-04-04' '15:42:02.481","puller_resolved_ts":"2025-04-04' '15:41:56.581","last_synced_ts":"2025-04-04' '15:41:56.630","now_ts":"2025-04-04' '15:42:03.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq .synced + status=false + '[' false '!=' false ']' ++ echo '{"synced":false,"sink_checkpoint_ts":"2025-04-04' '15:42:02.481","puller_resolved_ts":"2025-04-04' '15:41:56.581","last_synced_ts":"2025-04-04' '15:41:56.630","now_ts":"2025-04-04' '15:42:03.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq -r .info + info='The data syncing is not finished, please wait' + target_message='The data syncing is not finished, please wait' + '[' 'The data syncing is not finished, please wait' '!=' 'The data syncing is not finished, please wait' ']' + sleep 130 ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 221 100 221 0 0 7129 0 --:--:-- --:--:-- --:--:-- 7129 + synced_status='{"synced":true,"sink_checkpoint_ts":"2025-04-04 15:44:01.101","puller_resolved_ts":"2025-04-04 15:43:54.101","last_synced_ts":"2025-04-04 15:41:45.552","now_ts":"2025-04-04 15:44:02.000","info":"Data syncing is finished"}' ++ echo '{"synced":true,"sink_checkpoint_ts":"2025-04-04' '15:44:01.101","puller_resolved_ts":"2025-04-04' '15:43:54.101","last_synced_ts":"2025-04-04' '15:41:45.552","now_ts":"2025-04-04' '15:44:02.000","info":"Data' syncing is 'finished"}' ++ jq .synced + status=true + '[' true '!=' true ']' ++ echo '{"synced":true,"sink_checkpoint_ts":"2025-04-04' '15:44:01.101","puller_resolved_ts":"2025-04-04' '15:43:54.101","last_synced_ts":"2025-04-04' '15:41:45.552","now_ts":"2025-04-04' '15:44:02.000","info":"Data' syncing is 'finished"}' ++ jq -r .info + info='Data syncing is finished' + target_message='Data syncing is finished' + '[' 'Data syncing is finished' '!=' 'Data syncing is finished' ']' + cleanup_process cdc.test wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit + stop_tidb_cluster + run_case_with_failpoint conf/changefeed-redo.toml + rm -rf /tmp/tidb_cdc_test/synced_status_with_redo + mkdir -p /tmp/tidb_cdc_test/synced_status_with_redo + start_tidb_cluster --workdir /tmp/tidb_cdc_test/synced_status_with_redo shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory The 1 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/synced_status_with_redo Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 723 100 723 0 0 24100 0 --:--:-- --:--:-- --:--:-- 24100 + synced_status='{"synced":false,"sink_checkpoint_ts":"2025-04-04 15:42:03.481","puller_resolved_ts":"2025-04-04 15:42:03.481","last_synced_ts":"2025-04-04 15:41:56.630","now_ts":"2025-04-04 15:44:13.000","info":"Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' \u003e '\''Resolved-Ts'\'' \u003e '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait"}' ++ echo '{"synced":false,"sink_checkpoint_ts":"2025-04-04' '15:42:03.481","puller_resolved_ts":"2025-04-04' '15:42:03.481","last_synced_ts":"2025-04-04' '15:41:56.630","now_ts":"2025-04-04' '15:44:13.000","info":"Please' check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view ''\''TiKV-Details'\''' '\u003e' ''\''Resolved-Ts'\''' '\u003e' ''\''Max' Leader Resolved TS 'gap'\''' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please 'wait"}' ++ jq .synced + status=false + '[' false '!=' false ']' ++ echo '{"synced":false,"sink_checkpoint_ts":"2025-04-04' '15:42:03.481","puller_resolved_ts":"2025-04-04' '15:42:03.481","last_synced_ts":"2025-04-04' '15:41:56.630","now_ts":"2025-04-04' '15:44:13.000","info":"Please' check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view ''\''TiKV-Details'\''' '\u003e' ''\''Resolved-Ts'\''' '\u003e' ''\''Max' Leader Resolved TS 'gap'\''' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please 'wait"}' ++ jq -r .info + info='Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' + target_message='Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' + '[' 'Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' '!=' 'Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' ']' + cleanup_process cdc.test wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit + stop_tidb_cluster + run_case_with_unavailable_tidb conf/changefeed.toml + rm -rf /tmp/tidb_cdc_test/synced_status + mkdir -p /tmp/tidb_cdc_test/synced_status + start_tidb_cluster --workdir /tmp/tidb_cdc_test/synced_status shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory The 1 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/synced_status Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489360643397097389 TiDB Cluster ID. tikv_gc_leader_uuid 657ff085adc000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-9bld-w6hq9, pid:20527, start at 2025-04-04 15:44:24.775239655 +0800 CST m=+2.220764276 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:46:24.786 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:44:24.759 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:34:24.759 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489360643397097389 TiDB Cluster ID. tikv_gc_leader_uuid 657ff085adc000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-9bld-w6hq9, pid:20527, start at 2025-04-04 15:44:24.775239655 +0800 CST m=+2.220764276 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:46:24.786 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:44:24.759 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:34:24.759 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489360635972112812 TiDB Cluster ID. tikv_gc_leader_uuid 657ff085ad4000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-9bld-w6hq9, pid:20589, start at 2025-04-04 15:44:24.798506609 +0800 CST m=+2.161711437 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:46:24.811 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:44:24.807 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:34:24.807 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/synced_status_with_redo/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/synced_status_with_redo/tiflash/log/error.log arg matches is ArgMatches { args: {"pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/synced_status_with_redo/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/synced_status_with_redo/tiflash/log/proxy.log"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/synced_status_with_redo/tiflash-proxy.toml"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + cd /tmp/tidb_cdc_test/synced_status_with_redo + export 'GO_FAILPOINTS=github.com/pingcap/tiflow/cdc/owner/ChangefeedOwnerNotUpdateCheckpoint=return(true)' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ChangefeedOwnerNotUpdateCheckpoint=return(true)' ++ run_cdc_cli_tso_query 127.0.0.1 2379 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status_with_redo.cli.21971.out cli tso query --pd=http://127.0.0.1:2379 Verifying downstream PD is started... + set +x + tso='457114299769552897 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 457114299769552897 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + start_ts=457114299769552897 + run_cdc_server --workdir /tmp/tidb_cdc_test/synced_status_with_redo --binary cdc.test [Fri Apr 4 15:44:30 CST 2025] <<<<<< START cdc server in synced_status_with_redo case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ChangefeedOwnerNotUpdateCheckpoint=return(true)' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status_with_redo.2199922001.out server --log-file /tmp/tidb_cdc_test/synced_status_with_redo/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/synced_status_with_redo/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 07:44:33 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [817 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7f4eece0-a36f-48d4-9064-b8c24dcfc8d8 {"id":"7f4eece0-a36f-48d4-9064-b8c24dcfc8d8","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752671} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffc1f9e4e5 7f4eece0-a36f-48d4-9064-b8c24dcfc8d8 /tidb/cdc/default/default/upstream/7489360643397097389 {"id":7489360643397097389,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7f4eece0-a36f-48d4-9064-b8c24dcfc8d8 {"id":"7f4eece0-a36f-48d4-9064-b8c24dcfc8d8","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752671} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffc1f9e4e5 7f4eece0-a36f-48d4-9064-b8c24dcfc8d8 /tidb/cdc/default/default/upstream/7489360643397097389 {"id":7489360643397097389,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7f4eece0-a36f-48d4-9064-b8c24dcfc8d8 {"id":"7f4eece0-a36f-48d4-9064-b8c24dcfc8d8","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752671} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffc1f9e4e5 7f4eece0-a36f-48d4-9064-b8c24dcfc8d8 /tidb/cdc/default/default/upstream/7489360643397097389 {"id":7489360643397097389,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + config_path=conf/changefeed-redo.toml + SINK_URI='mysql://root@127.0.0.1:3306/?max-txn-row=1' + run_cdc_cli changefeed create --start-ts=457114299769552897 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/synced_status_with_redo/conf/changefeed-redo.toml + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status_with_redo.cli.22049.out cli changefeed create --start-ts=457114299769552897 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/synced_status_with_redo/conf/changefeed-redo.toml Create changefeed successfully! ID: test-1 Info: {"upstream_id":7489360643397097389,"namespace":"default","id":"test-1","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2025-04-04T15:44:34.353343875+08:00","start_ts":457114299769552897,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"eventual","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"storage":"file:///tmp/tidb_cdc_test/synced_status/redo","use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":120,"checkpoint_interval":20}},"state":"normal","creator_version":"v9.0.0-beta.1-10-ga306f4894","resolved_ts":457114299769552897,"checkpoint_ts":457114299769552897,"checkpoint_time":"2025-04-04 15:44:29.409"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489360683553478684 TiDB Cluster ID. tikv_gc_leader_uuid 657ff08f1cc0007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-qg8g-pl0pp, pid:23980, start at 2025-04-04 15:44:34.426596941 +0800 CST m=+1.598994485 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:46:34.434 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:44:34.419 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:34:34.419 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489360683553478684 TiDB Cluster ID. tikv_gc_leader_uuid 657ff08f1cc0007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-qg8g-pl0pp, pid:23980, start at 2025-04-04 15:44:34.426596941 +0800 CST m=+1.598994485 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:46:34.434 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:44:34.419 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:34:34.419 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489360688734181468 TiDB Cluster ID. tikv_gc_leader_uuid 657ff08f274000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-qg8g-pl0pp, pid:24057, start at 2025-04-04 15:44:34.502504693 +0800 CST m=+1.593350108 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:46:34.510 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:44:34.510 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:34:34.510 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/synced_status/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/synced_status/tiflash/log/error.log arg matches is ArgMatches { args: {"addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash/log/proxy.log"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash/db/proxy"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash-proxy.toml"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + set +x + sleep 20 + cd /tmp/tidb_cdc_test/synced_status ++ run_cdc_cli_tso_query 127.0.0.1 2379 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.25503.out cli tso query --pd=http://127.0.0.1:2379 + set +x + tso='457114301908647937 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 457114301908647937 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + start_ts=457114301908647937 + run_cdc_server --workdir /tmp/tidb_cdc_test/synced_status --binary cdc.test [Fri Apr 4 15:44:39 CST 2025] <<<<<< START cdc server in synced_status case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.2555425556.out server --log-file /tmp/tidb_cdc_test/synced_status/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/synced_status/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 07:44:42 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [817 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2d8e2756-1edf-4900-8fa7-202dc86ef7e6 {"id":"2d8e2756-1edf-4900-8fa7-202dc86ef7e6","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752679} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffc21e24d9 2d8e2756-1edf-4900-8fa7-202dc86ef7e6 /tidb/cdc/default/default/upstream/7489360683553478684 {"id":7489360683553478684,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2d8e2756-1edf-4900-8fa7-202dc86ef7e6 {"id":"2d8e2756-1edf-4900-8fa7-202dc86ef7e6","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752679} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffc21e24d9 2d8e2756-1edf-4900-8fa7-202dc86ef7e6 /tidb/cdc/default/default/upstream/7489360683553478684 {"id":7489360683553478684,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2d8e2756-1edf-4900-8fa7-202dc86ef7e6 {"id":"2d8e2756-1edf-4900-8fa7-202dc86ef7e6","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752679} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffc21e24d9 2d8e2756-1edf-4900-8fa7-202dc86ef7e6 /tidb/cdc/default/default/upstream/7489360683553478684 {"id":7489360683553478684,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + config_path=conf/changefeed.toml + SINK_URI='mysql://root@127.0.0.1:3306/?max-txn-row=1' + run_cdc_cli changefeed create --start-ts=457114301908647937 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/synced_status/conf/changefeed.toml + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.25605.out cli changefeed create --start-ts=457114301908647937 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/synced_status/conf/changefeed.toml Create changefeed successfully! ID: test-1 Info: {"upstream_id":7489360683553478684,"namespace":"default","id":"test-1","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2025-04-04T15:44:42.679338274+08:00","start_ts":457114301908647937,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":120,"checkpoint_interval":20}},"state":"normal","creator_version":"v9.0.0-beta.1-10-ga306f4894","resolved_ts":457114301908647937,"checkpoint_ts":457114301908647937,"checkpoint_time":"2025-04-04 15:44:37.569"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x + run_sql 'USE TEST;Create table t1(a int primary key, b int);insert into t1 values(1,2);insert into t1 values(2,3);' + check_table_exists test.t1 127.0.0.1 3306 table test.t1 not exists for 1-th check, retry later table test.t1 exists + sleep 5 + kill_tidb ++ ps aux ++ grep tidb-server ++ grep /tmp/tidb_cdc_test/synced_status + info='jenkins 23980 12.2 0.0 3258336 212276 ? Sl 15:44 0:02 tidb-server -P 4000 -config /tmp/tidb_cdc_test/synced_status/tidb-config-1743752672815685752.toml --store tikv --path 127.0.0.1:2379 --status=10080 --log-file /tmp/tidb_cdc_test/synced_status/tidb.log jenkins 23989 5.0 0.0 3331748 182976 ? Sl 15:44 0:00 tidb-server -P 4001 -config /tmp/tidb_cdc_test/synced_status/tidb-config-1743752672823828769.toml --store tikv --path 127.0.0.1:2379 --status=10081 --log-file /tmp/tidb_cdc_test/synced_status/tidb_other.log jenkins 24057 12.4 0.0 2733412 236244 ? Sl 15:44 0:02 tidb-server -P 3306 -config /tmp/tidb_cdc_test/synced_status/tidb-config-1743752672896476186.toml --store tikv --path 127.0.0.1:2479 --status=20080 --log-file /tmp/tidb_cdc_test/synced_status/tidb_down.log' ++ ps aux ++ grep tidb-server ++ grep /tmp/tidb_cdc_test/synced_status ++ awk '{print $2}' ++ xargs kill -9 ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 243 100 243 0 0 9720 0 --:--:-- --:--:-- --:--:-- 9720 + synced_status='{"synced":false,"sink_checkpoint_ts":"2025-04-04 15:44:50.868","puller_resolved_ts":"2025-04-04 15:44:44.119","last_synced_ts":"2025-04-04 15:44:44.119","now_ts":"2025-04-04 15:44:51.000","info":"The data syncing is not finished, please wait"}' ++ echo '{"synced":false,"sink_checkpoint_ts":"2025-04-04' '15:44:50.868","puller_resolved_ts":"2025-04-04' '15:44:44.119","last_synced_ts":"2025-04-04' '15:44:44.119","now_ts":"2025-04-04' '15:44:51.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq .synced + status=false + '[' false '!=' false ']' ++ echo '{"synced":false,"sink_checkpoint_ts":"2025-04-04' '15:44:50.868","puller_resolved_ts":"2025-04-04' '15:44:44.119","last_synced_ts":"2025-04-04' '15:44:44.119","now_ts":"2025-04-04' '15:44:51.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq -r .info + info='The data syncing is not finished, please wait' + target_message='The data syncing is not finished, please wait' + '[' 'The data syncing is not finished, please wait' '!=' 'The data syncing is not finished, please wait' ']' + sleep 130 ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 723 100 723 0 0 32863 0 --:--:-- --:--:-- --:--:-- 32863 + synced_status='{"synced":false,"sink_checkpoint_ts":"2025-04-04 15:44:29.409","puller_resolved_ts":"1970-01-01 08:00:00.000","last_synced_ts":"1970-01-01 08:00:00.000","now_ts":"2025-04-04 15:44:56.000","info":"Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' \u003e '\''Resolved-Ts'\'' \u003e '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait"}' ++ echo '{"synced":false,"sink_checkpoint_ts":"2025-04-04' '15:44:29.409","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2025-04-04' '15:44:56.000","info":"Please' check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view ''\''TiKV-Details'\''' '\u003e' ''\''Resolved-Ts'\''' '\u003e' ''\''Max' Leader Resolved TS 'gap'\''' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please 'wait"}' ++ jq .synced + status=false + '[' false '!=' false ']' ++ echo '{"synced":false,"sink_checkpoint_ts":"2025-04-04' '15:44:29.409","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2025-04-04' '15:44:56.000","info":"Please' check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view ''\''TiKV-Details'\''' '\u003e' ''\''Resolved-Ts'\''' '\u003e' ''\''Max' Leader Resolved TS 'gap'\''' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please 'wait"}' ++ jq -r .info + info='Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' + target_message='Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' + '[' 'Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' '!=' 'Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' ']' + export GO_FAILPOINTS= + GO_FAILPOINTS= + cleanup_process cdc.test wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit + stop_tidb_cluster + check_logs /tmp/tidb_cdc_test/synced_status_with_redo ++ date + echo '[Fri Apr 4 15:45:07 CST 2025] <<<<<< run test case synced_status_with_redo success! >>>>>>' [Fri Apr 4 15:45:07 CST 2025] <<<<<< run test case synced_status_with_redo success! >>>>>> + stop_tidb_cluster \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-3484/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 221 100 221 0 0 8185 0 --:--:-- --:--:-- --:--:-- 8185 + synced_status='{"synced":true,"sink_checkpoint_ts":"2025-04-04 15:47:00.019","puller_resolved_ts":"2025-04-04 15:46:53.019","last_synced_ts":"2025-04-04 15:44:44.119","now_ts":"2025-04-04 15:47:01.000","info":"Data syncing is finished"}' ++ echo '{"synced":true,"sink_checkpoint_ts":"2025-04-04' '15:47:00.019","puller_resolved_ts":"2025-04-04' '15:46:53.019","last_synced_ts":"2025-04-04' '15:44:44.119","now_ts":"2025-04-04' '15:47:01.000","info":"Data' syncing is 'finished"}' ++ jq .synced + status=true + '[' true '!=' true ']' ++ echo '{"synced":true,"sink_checkpoint_ts":"2025-04-04' '15:47:00.019","puller_resolved_ts":"2025-04-04' '15:46:53.019","last_synced_ts":"2025-04-04' '15:44:44.119","now_ts":"2025-04-04' '15:47:01.000","info":"Data' syncing is 'finished"}' ++ jq -r .info + info='Data syncing is finished' + target_message='Data syncing is finished' + '[' 'Data syncing is finished' '!=' 'Data syncing is finished' ']' + cleanup_process cdc.test wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit + stop_tidb_cluster + run_case_with_failpoint conf/changefeed.toml + rm -rf /tmp/tidb_cdc_test/synced_status + mkdir -p /tmp/tidb_cdc_test/synced_status + start_tidb_cluster --workdir /tmp/tidb_cdc_test/synced_status shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory The 1 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/synced_status Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489361396286412763 TiDB Cluster ID. tikv_gc_leader_uuid 657ff130dcc000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-qg8g-pl0pp, pid:26828, start at 2025-04-04 15:47:20.071089902 +0800 CST m=+1.518728283 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:49:20.079 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:47:20.051 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:37:20.051 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489361396286412763 TiDB Cluster ID. tikv_gc_leader_uuid 657ff130dcc000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-qg8g-pl0pp, pid:26828, start at 2025-04-04 15:47:20.071089902 +0800 CST m=+1.518728283 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:49:20.079 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:47:20.051 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:37:20.051 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489361402853374013 TiDB Cluster ID. tikv_gc_leader_uuid 657ff130f4c000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-3484-qg8g-pl0pp, pid:26911, start at 2025-04-04 15:47:20.180851513 +0800 CST m=+1.547651753 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250404-15:49:20.188 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250404-15:47:20.147 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250404-15:37:20.147 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/synced_status/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/synced_status/tiflash/log/error.log arg matches is ArgMatches { args: {"config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash-proxy.toml"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash/db/proxy"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash/log/proxy.log"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + cd /tmp/tidb_cdc_test/synced_status + export 'GO_FAILPOINTS=github.com/pingcap/tiflow/cdc/owner/ChangefeedOwnerNotUpdateCheckpoint=return(true)' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ChangefeedOwnerNotUpdateCheckpoint=return(true)' ++ run_cdc_cli_tso_query 127.0.0.1 2379 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.28347.out cli tso query --pd=http://127.0.0.1:2379 + set +x + tso='457114345354035201 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 457114345354035201 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + start_ts=457114345354035201 + run_cdc_server --workdir /tmp/tidb_cdc_test/synced_status --binary cdc.test [Fri Apr 4 15:47:24 CST 2025] <<<<<< START cdc server in synced_status case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ChangefeedOwnerNotUpdateCheckpoint=return(true)' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.2838928391.out server --log-file /tmp/tidb_cdc_test/synced_status/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/synced_status/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * connect to 127.0.0.1 port 8300 failed: Connection refused * Failed to connect to 127.0.0.1 port 8300: Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * Trying 127.0.0.1... * TCP_NODELAY set * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Host: 127.0.0.1:8300 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.61.1 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 04 Apr 2025 07:47:27 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [817 bytes data] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/35f045e1-94b4-4294-882d-c42543c6fc04 {"id":"35f045e1-94b4-4294-882d-c42543c6fc04","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752845} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffc4ab66dd 35f045e1-94b4-4294-882d-c42543c6fc04 /tidb/cdc/default/default/upstream/7489361396286412763 {"id":7489361396286412763,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/35f045e1-94b4-4294-882d-c42543c6fc04 {"id":"35f045e1-94b4-4294-882d-c42543c6fc04","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752845} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffc4ab66dd 35f045e1-94b4-4294-882d-c42543c6fc04 /tidb/cdc/default/default/upstream/7489361396286412763 {"id":7489361396286412763,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/35f045e1-94b4-4294-882d-c42543c6fc04 {"id":"35f045e1-94b4-4294-882d-c42543c6fc04","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-10-ga306f4894","git-hash":"a306f48949e037965a7c1963f500f7e0abbfba51","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1743752845} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223195ffc4ab66dd 35f045e1-94b4-4294-882d-c42543c6fc04 /tidb/cdc/default/default/upstream/7489361396286412763 {"id":7489361396286412763,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + config_path=conf/changefeed.toml + SINK_URI='mysql://root@127.0.0.1:3306/?max-txn-row=1' + run_cdc_cli changefeed create --start-ts=457114345354035201 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/synced_status/conf/changefeed.toml + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.28442.out cli changefeed create --start-ts=457114345354035201 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/synced_status/conf/changefeed.toml Create changefeed successfully! ID: test-1 Info: {"upstream_id":7489361396286412763,"namespace":"default","id":"test-1","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2025-04-04T15:47:28.192102328+08:00","start_ts":457114345354035201,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":120,"checkpoint_interval":20}},"state":"normal","creator_version":"v9.0.0-beta.1-10-ga306f4894","resolved_ts":457114345354035201,"checkpoint_ts":457114345354035201,"checkpoint_time":"2025-04-04 15:47:23.300"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x + sleep 20 ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 723 100 723 0 0 20083 0 --:--:-- --:--:-- --:--:-- 20083 + synced_status='{"synced":false,"sink_checkpoint_ts":"2025-04-04 15:47:23.300","puller_resolved_ts":"1970-01-01 08:00:00.000","last_synced_ts":"1970-01-01 08:00:00.000","now_ts":"2025-04-04 15:47:49.000","info":"Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' \u003e '\''Resolved-Ts'\'' \u003e '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait"}' ++ echo '{"synced":false,"sink_checkpoint_ts":"2025-04-04' '15:47:23.300","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2025-04-04' '15:47:49.000","info":"Please' check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view ''\''TiKV-Details'\''' '\u003e' ''\''Resolved-Ts'\''' '\u003e' ''\''Max' Leader Resolved TS 'gap'\''' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please 'wait"}' ++ jq .synced + status=false + '[' false '!=' false ']' ++ echo '{"synced":false,"sink_checkpoint_ts":"2025-04-04' '15:47:23.300","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2025-04-04' '15:47:49.000","info":"Please' check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view ''\''TiKV-Details'\''' '\u003e' ''\''Resolved-Ts'\''' '\u003e' ''\''Max' Leader Resolved TS 'gap'\''' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please 'wait"}' ++ jq -r .info + info='Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' + target_message='Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' + '[' 'Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' '!=' 'Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' ']' + export GO_FAILPOINTS= + GO_FAILPOINTS= + cleanup_process cdc.test wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit + stop_tidb_cluster + check_logs /tmp/tidb_cdc_test/synced_status ++ date + echo '[Fri Apr 4 15:48:00 CST 2025] <<<<<< run test case synced_status success! >>>>>>' [Fri Apr 4 15:48:00 CST 2025] <<<<<< run test case synced_status success! >>>>>> + stop_tidb_cluster \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-3484/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // parallel [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] End of Pipeline Finished: SUCCESS