Started by user Jenkins Admin Obtained pipelines/pingcap/tiflow/latest/periodics_cdc_integration_mysql_test.groovy from git https://github.com/PingCAP-QE/ci.git Loading library tipipeline@main Library tipipeline@main is cached. Copying from home. [Pipeline] Start of Pipeline [Pipeline] readTrusted Obtained pipelines/pingcap/tiflow/latest/pod-periodics_cdc_integration_mysql_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-periodics-cdc-integration-mysql-test-234-0-p62hf Agent pingcap-tiflow-periodics-cdc-integration-mysql-test-234-0-p62hf is provisioned from template pingcap_tiflow_periodics_cdc_integration_mysql_test_234-080mz-xvzmt --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/" runUrl: "job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "74d983b7ee9b8bf86908571bbab2798c7b4c6da3" jenkins/label: "pingcap_tiflow_periodics_cdc_integration_mysql_test_234-080mz" name: "pingcap-tiflow-periodics-cdc-integration-mysql-test-234-0-p62hf" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.23:latest" name: "golang" resources: limits: memory: "32Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-periodics-cdc-integration-mysql-test-234-0-p62hf" - name: "JENKINS_NAME" value: "pingcap-tiflow-periodics-cdc-integration-mysql-test-234-0-p62hf" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-periodics-cdc-integration-mysql-test-234-0-p62hf in /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test [Pipeline] { [Pipeline] stage [Pipeline] { (Declarative: Checkout SCM) [Pipeline] checkout The recommended git tool is: git No credentials specified Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Avoid second fetch Checking out Revision b2a59508c11c86ba701c4af25fc536a9475b82c6 (origin/main) > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f b2a59508c11c86ba701c4af25fc536a9475b82c6 # timeout=10 Commit message: "feat(prow-jobs/pingcap-qe/ci): add periodic job to crawl pull requests and issues (#3449)" > git rev-list --no-walk b2a59508c11c86ba701c4af25fc536a9475b82c6 # timeout=10 [Pipeline] } [Pipeline] // stage [Pipeline] withEnv [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] timeout Timeout set to expire in 1 hr 5 min [Pipeline] { [Pipeline] stage [Pipeline] { (Debug info) [Pipeline] sh + printenv PROW_JOB_ID=1ff7fcb7-127a-11f0-bebb-96e16cfa81f4 JENKINS_NODE_COOKIE=a68fab21-d544-4bdc-9e96-3671e8ce47fb BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/ GOLANG_VERSION=1.23.3 HOSTNAME=pingcap-tiflow-periodics-cdc-integration-mysql-test-234-0-p62hf HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Debug info BUILD_TAG=jenkins-pingcap-tiflow-periodics_cdc_integration_mysql_test-234 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 JOB_SPEC={"type":"periodic","job":"pingcap/tiflow/periodics_cdc_integration_mysql_test","buildid":"1908671202193313792","prowjobid":"1ff7fcb7-127a-11f0-bebb-96e16cfa81f4","extra_refs":[{"org":"pingcap","repo":"tiflow","base_ref":"master"}]} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct _=/usr/bin/printenv POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test HUDSON_URL=https://do.pingcap.net/jenkins/ JOB_NAME=pingcap/tiflow/periodics_cdc_integration_mysql_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#234 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1908671202193313792 GOLANG_DOWNLOAD_SHA256=a0afb9744c00648bafb1b90b4aba5bdb86f424f02f9275399ce0c20b93a2c3a8 JOB_BASE_NAME=periodics_cdc_integration_mysql_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/display/redirect?page=tests SHLVL=3 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_periodics_cdc_integration_mysql_test_234-080mz GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-a0e5b959a55caf4285c3f0608dcbd47ed90740dac7e4244afe4990642ac521c1 NODE_LABELS=pingcap_tiflow_periodics_cdc_integration_mysql_test_234-080mz pingcap-tiflow-periodics-cdc-integration-mysql-test-234-0-p62hf GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-periodics-cdc-integration-mysql-test-234-0-p62hf GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/display/redirect BUILD_NUMBER=234 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.23.3.linux-amd64.tar.gz + echo ------------------------- ------------------------- + go env GO111MODULE='' GOARCH='amd64' GOBIN='' GOCACHE='/home/jenkins/.cache/go-build' GOENV='/home/jenkins/.config/go/env' GOEXE='' GOEXPERIMENT='' GOFLAGS='' GOHOSTARCH='amd64' GOHOSTOS='linux' GOINSECURE='' GOMODCACHE='/go/pkg/mod' GONOPROXY='' GONOSUMDB='' GOOS='linux' GOPATH='/go' GOPRIVATE='' GOPROXY='http://goproxy.apps.svc,https://proxy.golang.org,direct' GOROOT='/usr/local/go' GOSUMDB='sum.golang.org' GOTMPDIR='' GOTOOLCHAIN='auto' GOTOOLDIR='/usr/local/go/pkg/tool/linux_amd64' GOVCS='' GOVERSION='go1.23.3' GODEBUG='' GOTELEMETRY='local' GOTELEMETRYDIR='/home/jenkins/.config/go/telemetry' GCCGO='gccgo' GOAMD64='v1' AR='ar' CC='gcc' CXX='g++' CGO_ENABLED='1' GOMOD='/dev/null' GOWORK='' CGO_CFLAGS='-O2 -g' CGO_CPPFLAGS='' CGO_CXXFLAGS='-O2 -g' CGO_FFLAGS='-O2 -g' CGO_LDFLAGS='-O2 -g' PKG_CONFIG='pkg-config' GOGCCFLAGS='-fPIC -m64 -pthread -Wl,--no-gc-sections -fmessage-length=0 -fdebug-prefix-map=/tmp/go-build3384556920=/tmp/go-build -gno-record-gcc-switches' + echo ------------------------- ------------------------- + echo 'debug command: kubectl -n jenkins-tiflow exec -ti pingcap-tiflow-periodics-cdc-integration-mysql-test-234-0-p62hf bash' debug command: kubectl -n jenkins-tiflow exec -ti pingcap-tiflow-periodics-cdc-integration-mysql-test-234-0-p62hf bash [Pipeline] container [Pipeline] { [Pipeline] sh + dig github.com ; <<>> DiG 9.18.16 <<>> github.com ;; global options: +cmd ;; Got answer: ;; ->>HEADER<<- opcode: QUERY, status: NOERROR, id: 53683 ;; flags: qr aa rd ra; QUERY: 1, ANSWER: 2, AUTHORITY: 0, ADDITIONAL: 1 ;; OPT PSEUDOSECTION: ; EDNS: version: 0, flags:; udp: 1232 ; COOKIE: a0ca5711e586ffaa (echoed) ;; QUESTION SECTION: ;github.com. IN A ;; ANSWER SECTION: github.com. 26 IN A 20.27.177.113 github.com. 26 IN A 20.205.243.166 ;; Query time: 92 msec ;; SERVER: 169.254.25.10#53(169.254.25.10) (UDP) ;; WHEN: Sun Apr 06 00:01:15 UTC 2025 ;; MSG SIZE rcvd: 103 [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // stage [Pipeline] stage [Pipeline] { (Checkout) [Pipeline] timeout Timeout set to expire in 10 min [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow [Pipeline] { [Pipeline] cache Cache restored successfully (git/pingcap/tiflow/rev-master) 192245760 bytes in 0.97 secs (197860154 bytes/sec) [Pipeline] { [Pipeline] retry [Pipeline] { [Pipeline] script [Pipeline] { [Pipeline] echo computeBranchFromPR component: tiflow, prTargetBranch: master, prTitle: , trunkBranch: master [Pipeline] sh git version 2.36.6 Reinitialized existing Git repository in /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/.git/ .git HEAD is now at 4c631d595 kafka(ticdc): ddl sink close the underline sink if send ddl or checkpoint failed and refactor the kafka ddl sink (#12112) POST git-upload-pack (383 bytes) From https://github.com/pingcap/tiflow = [up to date] master -> origin/master HEAD is now at 4c631d595 kafka(ticdc): ddl sink close the underline sink if send ddl or checkpoint failed and refactor the kafka ddl sink (#12112) 4c631d5951854ac91765b729fe3849d8e8d5ed8a 4c631d595 kafka(ticdc): ddl sink close the underline sink if send ddl or checkpoint failed and refactor the kafka ddl sink (#12112) 3cf60079e codec(ticdc): modify DDL type support about debezium protocol (#12113) 9ed3d76c5 consumer(ticdc): adjust consumer log level (#12092) ✅ Checkout master 🎉 /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow ✅ ~~~~~ All done. ~~~~~~ [Pipeline] } [Pipeline] // script [Pipeline] } [Pipeline] // retry [Pipeline] } Cache not saved (git/pingcap/tiflow/rev-master already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] stage [Pipeline] { (prepare) [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/third_party_download [Pipeline] { [Pipeline] script [Pipeline] { [Pipeline] retry [Pipeline] { [Pipeline] sh + export TIDB_BRANCH=master + TIDB_BRANCH=master + export PD_BRANCH=master + PD_BRANCH=master + export TIKV_BRANCH=master + TIKV_BRANCH=master + export TIFLASH_BRANCH=master + TIFLASH_BRANCH=master + cd ../tiflow + ./scripts/download-integration-test-binaries.sh master Downloading binaries... >>> Downloading tidb-server.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/tidb/d9fb3a53376157be49b5a7941e6e689a33c81eb0/centos7/tidb-server.tar.gz 2025-04-06 08:01:50 URL:http://fileserver.pingcap.net/download/builds/pingcap/tidb/d9fb3a53376157be49b5a7941e6e689a33c81eb0/centos7/tidb-server.tar.gz [620275338/620275338] -> "tmp/tidb-server.tar.gz" [1] >>> Downloading pd-server.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/pd/9d6bbb8529d9d2acb9cf324a65e94ece465f6346/centos7/pd-server.tar.gz 2025-04-06 08:02:09 URL:http://fileserver.pingcap.net/download/builds/pingcap/pd/9d6bbb8529d9d2acb9cf324a65e94ece465f6346/centos7/pd-server.tar.gz [228444265/228444265] -> "tmp/pd-server.tar.gz" [1] >>> Downloading tikv-server.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/tikv/3eb3439fd88f0db9de06ef92e845e80b32c9b49a/centos7/tikv-server.tar.gz 2025-04-06 08:02:40 URL:http://fileserver.pingcap.net/download/builds/pingcap/tikv/3eb3439fd88f0db9de06ef92e845e80b32c9b49a/centos7/tikv-server.tar.gz [817400808/817400808] -> "tmp/tikv-server.tar.gz" [1] >>> Downloading tiflash.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/tiflash/master/8ecb79fcf30098e6786f4d0440ead2a879a3ad4a/centos7/tiflash.tar.gz 2025-04-06 08:03:08 URL:http://fileserver.pingcap.net/download/builds/pingcap/tiflash/master/8ecb79fcf30098e6786f4d0440ead2a879a3ad4a/centos7/tiflash.tar.gz [553235170/553235170] -> "tmp/tiflash.tar.gz" [1] >>> Downloading minio.tar.gz from http://fileserver.pingcap.net/download/minio.tar.gz 2025-04-06 08:03:15 URL:http://fileserver.pingcap.net/download/minio.tar.gz [17718777/17718777] -> "tmp/minio.tar.gz" [1] >>> Downloading etcd.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/cdc/etcd-v3.4.7-linux-amd64.tar.gz 2025-04-06 08:03:15 URL:http://fileserver.pingcap.net/download/builds/pingcap/cdc/etcd-v3.4.7-linux-amd64.tar.gz [17310840/17310840] -> "tmp/etcd.tar.gz" [1] >>> Downloading schema-registry.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/cdc/schema-registry.tar.gz 2025-04-06 08:03:25 URL:http://fileserver.pingcap.net/download/builds/pingcap/cdc/schema-registry.tar.gz [278386006/278386006] -> "tmp/schema-registry.tar.gz" [1] >>> Downloading go-ycsb from http://fileserver.pingcap.net/download/builds/pingcap/go-ycsb/test-br/go-ycsb 2025-04-06 08:03:29 URL:http://fileserver.pingcap.net/download/builds/pingcap/go-ycsb/test-br/go-ycsb [45975512/45975512] -> "third_bin/go-ycsb" [1] >>> Downloading jq from http://fileserver.pingcap.net/download/builds/pingcap/test/jq-1.6/jq-linux64 2025-04-06 08:03:29 URL:http://fileserver.pingcap.net/download/builds/pingcap/test/jq-1.6/jq-linux64 [3953824/3953824] -> "third_bin/jq" [1] Download SUCCESS + ls -alh ./bin total 2.0G drwxr-sr-x 6 jenkins jenkins 4.0K Apr 6 08:03 . drwxr-sr-x 19 jenkins jenkins 4.0K Apr 6 08:03 .. drwxr-sr-x 2 jenkins jenkins 4.0K May 19 2023 bin drwxr-sr-x 4 jenkins jenkins 4.0K May 10 2023 etc -rwxr-xr-x 1 jenkins jenkins 17M Apr 2 2020 etcdctl -rwxr-xr-x 1 jenkins jenkins 44M Apr 6 08:03 go-ycsb -rwxr-xr-x 1 jenkins jenkins 3.8M Apr 6 08:03 jq drwxr-sr-x 3 jenkins jenkins 4.0K May 10 2023 lib lrwxrwxrwx 1 jenkins jenkins 13 Apr 3 12:33 libc++.so.1 -> libc++.so.1.0 -rwxr-xr-x 1 jenkins jenkins 1.2M Dec 18 2023 libc++.so.1.0 lrwxrwxrwx 1 jenkins jenkins 16 Apr 3 12:33 libc++abi.so.1 -> libc++abi.so.1.0 -rwxr-xr-x 1 jenkins jenkins 353K Dec 18 2023 libc++abi.so.1.0 lrwxrwxrwx 1 jenkins jenkins 13 Apr 3 12:33 libgmssl.so -> libgmssl.so.3 lrwxrwxrwx 1 jenkins jenkins 15 Apr 3 12:33 libgmssl.so.3 -> libgmssl.so.3.0 -rwxr-xr-x 1 jenkins jenkins 2.2M Apr 3 11:49 libgmssl.so.3.0 -rwxr-xr-x 1 jenkins jenkins 346M Apr 3 12:34 libtiflash_proxy.so -rwxr-xr-x 1 jenkins jenkins 50M Jul 29 2020 minio -rwxr-xr-x 1 jenkins jenkins 41M Apr 3 15:30 pd-api-bench -rwxr-xr-x 1 jenkins jenkins 44M Apr 3 15:29 pd-ctl -rwxr-xr-x 1 jenkins jenkins 40M Apr 3 15:30 pd-heartbeat-bench -rwxr-xr-x 1 jenkins jenkins 35M Apr 3 15:29 pd-recover -rwxr-xr-x 1 jenkins jenkins 37M Apr 3 15:30 pd-region-bench -rwxr-xr-x 1 jenkins jenkins 115M Apr 3 15:29 pd-server -rwxr-xr-x 1 jenkins jenkins 37M Apr 3 15:30 pd-tso-bench -rwxr-xr-x 1 jenkins jenkins 15M Apr 3 15:30 pd-ut -rwxr-xr-x 1 jenkins jenkins 35M Apr 3 15:30 regions-dump drwxr-sr-x 4 jenkins jenkins 4.0K May 10 2023 share -rwxr-xr-x 1 jenkins jenkins 35M Apr 3 15:30 stores-dump -rwxr-xr-x 1 jenkins jenkins 241M Apr 3 19:14 tidb-server -rwxr-xr-x 1 jenkins jenkins 443M Apr 3 12:33 tiflash -rwxr-xr-x 1 jenkins jenkins 373M Apr 2 11:56 tikv-server -rwxr-xr-x 1 jenkins jenkins 2.3M Apr 3 15:30 xprog + make check_third_party_binary go: downloading go1.23.4 (linux/amd64) /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/tidb-server /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/tikv-server /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/pd-server /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/tiflash /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/pd-ctl /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/go-ycsb /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/etcdctl /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/jq /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/minio /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/bin/schema-registry-start + cd - /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/third_party_download + mkdir -p bin + mv ../tiflow/bin/bin ../tiflow/bin/etc ../tiflow/bin/etcdctl ../tiflow/bin/go-ycsb ../tiflow/bin/jq ../tiflow/bin/lib ../tiflow/bin/libc++.so.1 ../tiflow/bin/libc++.so.1.0 ../tiflow/bin/libc++abi.so.1 ../tiflow/bin/libc++abi.so.1.0 ../tiflow/bin/libgmssl.so ../tiflow/bin/libgmssl.so.3 ../tiflow/bin/libgmssl.so.3.0 ../tiflow/bin/libtiflash_proxy.so ../tiflow/bin/minio ../tiflow/bin/pd-api-bench ../tiflow/bin/pd-ctl ../tiflow/bin/pd-heartbeat-bench ../tiflow/bin/pd-recover ../tiflow/bin/pd-region-bench ../tiflow/bin/pd-server ../tiflow/bin/pd-tso-bench ../tiflow/bin/pd-ut ../tiflow/bin/regions-dump ../tiflow/bin/share ../tiflow/bin/stores-dump ../tiflow/bin/tidb-server ../tiflow/bin/tiflash ../tiflow/bin/tikv-server ../tiflow/bin/xprog ./bin/ + ls -alh ./bin total 2.0G drwxr-sr-x 6 jenkins jenkins 4.0K Apr 6 08:03 . drwxr-sr-x 3 jenkins jenkins 4.0K Apr 6 08:03 .. drwxr-sr-x 2 jenkins jenkins 4.0K May 19 2023 bin drwxr-sr-x 4 jenkins jenkins 4.0K May 10 2023 etc -rwxr-xr-x 1 jenkins jenkins 17M Apr 2 2020 etcdctl -rwxr-xr-x 1 jenkins jenkins 44M Apr 6 08:03 go-ycsb -rwxr-xr-x 1 jenkins jenkins 3.8M Apr 6 08:03 jq drwxr-sr-x 3 jenkins jenkins 4.0K May 10 2023 lib lrwxrwxrwx 1 jenkins jenkins 13 Apr 3 12:33 libc++.so.1 -> libc++.so.1.0 -rwxr-xr-x 1 jenkins jenkins 1.2M Dec 18 2023 libc++.so.1.0 lrwxrwxrwx 1 jenkins jenkins 16 Apr 3 12:33 libc++abi.so.1 -> libc++abi.so.1.0 -rwxr-xr-x 1 jenkins jenkins 353K Dec 18 2023 libc++abi.so.1.0 lrwxrwxrwx 1 jenkins jenkins 13 Apr 3 12:33 libgmssl.so -> libgmssl.so.3 lrwxrwxrwx 1 jenkins jenkins 15 Apr 3 12:33 libgmssl.so.3 -> libgmssl.so.3.0 -rwxr-xr-x 1 jenkins jenkins 2.2M Apr 3 11:49 libgmssl.so.3.0 -rwxr-xr-x 1 jenkins jenkins 346M Apr 3 12:34 libtiflash_proxy.so -rwxr-xr-x 1 jenkins jenkins 50M Jul 29 2020 minio -rwxr-xr-x 1 jenkins jenkins 41M Apr 3 15:30 pd-api-bench -rwxr-xr-x 1 jenkins jenkins 44M Apr 3 15:29 pd-ctl -rwxr-xr-x 1 jenkins jenkins 40M Apr 3 15:30 pd-heartbeat-bench -rwxr-xr-x 1 jenkins jenkins 35M Apr 3 15:29 pd-recover -rwxr-xr-x 1 jenkins jenkins 37M Apr 3 15:30 pd-region-bench -rwxr-xr-x 1 jenkins jenkins 115M Apr 3 15:29 pd-server -rwxr-xr-x 1 jenkins jenkins 37M Apr 3 15:30 pd-tso-bench -rwxr-xr-x 1 jenkins jenkins 15M Apr 3 15:30 pd-ut -rwxr-xr-x 1 jenkins jenkins 35M Apr 3 15:30 regions-dump drwxr-sr-x 4 jenkins jenkins 4.0K May 10 2023 share -rwxr-xr-x 1 jenkins jenkins 35M Apr 3 15:30 stores-dump -rwxr-xr-x 1 jenkins jenkins 241M Apr 3 19:14 tidb-server -rwxr-xr-x 1 jenkins jenkins 443M Apr 3 12:33 tiflash -rwxr-xr-x 1 jenkins jenkins 373M Apr 2 11:56 tikv-server -rwxr-xr-x 1 jenkins jenkins 2.3M Apr 3 15:30 xprog + ./bin/tidb-server -V Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore + ./bin/pd-server -V Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 + ./bin/tikv-server -V TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + ./bin/tiflash --version TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored [Pipeline] } [Pipeline] // retry [Pipeline] } [Pipeline] // script [Pipeline] } [Pipeline] // dir [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow [Pipeline] { [Pipeline] sh + ls -alh ./bin total 8.0K drwxr-sr-x 2 jenkins jenkins 4.0K Apr 6 08:03 . drwxr-sr-x 19 jenkins jenkins 4.0K Apr 6 08:03 .. + '[' -f ./bin/cdc ']' + make cdc CGO_ENABLED=0 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v9.0.0-beta.1-9-g4c631d595" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2025-04-06 00:03:35" -X "github.com/pingcap/tiflow/pkg/version.GitHash=4c631d5951854ac91765b729fe3849d8e8d5ed8a" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v9.0.0-beta.1-9-g4c631d595"' -o bin/cdc ./cmd/cdc go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20241223052309-3735ed55a394 go: downloading github.com/pingcap/errors v0.11.5-0.20240318064555-6bd07397691f go: downloading github.com/pingcap/failpoint v0.0.0-20240528011301-b51a646c7c86 go: downloading github.com/fatih/color v1.18.0 go: downloading go.uber.org/zap v1.27.0 go: downloading github.com/pingcap/log v1.1.1-0.20241212030209-7e3ff8601a2a go: downloading github.com/spf13/pflag v1.0.5 go: downloading github.com/tikv/pd/client v0.0.0-20241111073742-238d4d79ea31 go: downloading github.com/xdg/scram v1.0.5 go: downloading google.golang.org/grpc v1.64.0 go: downloading golang.org/x/sync v0.10.0 go: downloading github.com/go-sql-driver/mysql v1.7.1 go: downloading github.com/prometheus/client_golang v1.20.5 go: downloading github.com/coreos/go-semver v0.3.1 go: downloading github.com/pingcap/kvproto v0.0.0-20241120022153-92b0414aeed8 go: downloading github.com/IBM/sarama v1.41.2 go: downloading github.com/gin-gonic/gin v1.9.1 go: downloading github.com/tinylib/msgp v1.1.6 go: downloading github.com/spf13/cobra v1.8.1 go: downloading github.com/google/btree v1.1.2 go: downloading go.uber.org/multierr v1.11.0 go: downloading github.com/BurntSushi/toml v1.4.1-0.20240526193622-a339e1f7089c go: downloading golang.org/x/net v0.33.0 go: downloading github.com/dustin/go-humanize v1.0.1 go: downloading github.com/pingcap/sysutil v1.0.1-0.20240311050922-ae81ee01f3a5 go: downloading github.com/tikv/client-go/v2 v2.0.8-0.20241209094930-06d7f4b9233b go: downloading github.com/gogo/protobuf v1.3.2 go: downloading github.com/DATA-DOG/go-sqlmock v1.5.0 go: downloading github.com/imdario/mergo v0.3.16 go: downloading github.com/pingcap/tidb/pkg/parser v0.0.0-20241223052309-3735ed55a394 go: downloading github.com/apache/pulsar-client-go v0.11.0 go: downloading github.com/aws/aws-sdk-go-v2 v1.19.1 go: downloading cloud.google.com/go/storage v1.39.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v1.16.0 go: downloading github.com/KimMachineGun/automemlimit v0.2.4 go: downloading github.com/aws/aws-sdk-go v1.55.5 go: downloading github.com/json-iterator/go v1.1.12 go: downloading github.com/shirou/gopsutil/v3 v3.24.5 go: downloading github.com/klauspost/compress v1.17.9 go: downloading github.com/pierrec/lz4/v4 v4.1.18 go: downloading github.com/robfig/cron v1.2.0 go: downloading github.com/hashicorp/golang-lru v0.5.1 go: downloading github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 go: downloading golang.org/x/oauth2 v0.24.0 go: downloading github.com/stretchr/testify v1.10.0 go: downloading github.com/uber-go/atomic v1.4.0 go: downloading go.etcd.io/etcd/pkg/v3 v3.5.12 go: downloading github.com/swaggo/files v0.0.0-20210815190702-a29dd2bc99b2 go: downloading github.com/swaggo/gin-swagger v1.2.0 go: downloading github.com/cockroachdb/pebble v1.1.0 go: downloading github.com/benbjohnson/clock v1.1.0 go: downloading github.com/tikv/pd v1.1.0-beta.0.20240407022249-7179657d129b go: downloading go.etcd.io/etcd/api/v3 v3.5.12 go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.12 go: downloading go.etcd.io/etcd/client/v3 v3.5.12 go: downloading go.etcd.io/etcd/server/v3 v3.5.12 go: downloading go.uber.org/atomic v1.11.0 go: downloading cloud.google.com/go v0.112.2 go: downloading github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 go: downloading github.com/jcmturner/gokrb5/v8 v8.4.4 go: downloading github.com/cenkalti/backoff/v4 v4.2.1 go: downloading golang.org/x/time v0.7.0 go: downloading github.com/segmentio/kafka-go v0.4.41-0.20230526171612-f057b1d369cd go: downloading github.com/soheilhy/cmux v0.1.5 go: downloading github.com/gavv/monotime v0.0.0-20190418164738-30dba4353424 go: downloading github.com/r3labs/diff v1.1.0 go: downloading github.com/goccy/go-json v0.10.2 go: downloading golang.org/x/sys v0.28.0 go: downloading github.com/google/uuid v1.6.0 go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 go: downloading github.com/go-mysql-org/go-mysql v1.9.1 go: downloading github.com/pingcap/check v0.0.0-20211026125417-57bd13f7b5f0 go: downloading github.com/mattn/go-colorable v0.1.13 go: downloading github.com/mattn/go-isatty v0.0.20 go: downloading gopkg.in/natefinch/lumberjack.v2 v2.2.1 go: downloading github.com/xdg/stringprep v1.0.3 go: downloading golang.org/x/crypto v0.31.0 go: downloading github.com/containerd/cgroups v1.0.4 go: downloading github.com/swaggo/swag v1.16.3 go: downloading github.com/golang/mock v1.6.0 go: downloading github.com/philhofer/fwd v1.1.1 go: downloading github.com/pingcap/tidb-dashboard v0.0.0-20240326110213-9768844ff5d7 go: downloading github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc go: downloading github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 go: downloading github.com/stretchr/objx v0.5.2 go: downloading github.com/gin-contrib/sse v0.1.0 go: downloading github.com/go-playground/validator/v10 v10.14.0 go: downloading github.com/pelletier/go-toml/v2 v2.0.8 go: downloading github.com/ugorji/go/codec v1.2.11 go: downloading google.golang.org/protobuf v1.35.2 go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd go: downloading github.com/golang/protobuf v1.5.4 go: downloading github.com/coreos/go-systemd/v22 v22.5.0 go: downloading github.com/prometheus/common v0.61.0 go: downloading github.com/prometheus/client_model v0.6.1 go: downloading github.com/beorn7/perks v1.0.1 go: downloading github.com/cespare/xxhash/v2 v2.3.0 go: downloading github.com/prometheus/procfs v0.15.1 go: downloading github.com/aws/smithy-go v1.13.5 go: downloading github.com/eapache/go-resiliency v1.4.0 go: downloading github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 go: downloading github.com/eapache/queue v1.1.0 go: downloading github.com/hashicorp/go-multierror v1.1.1 go: downloading github.com/jcmturner/gofork v1.7.6 go: downloading github.com/bits-and-blooms/bitset v1.14.3 go: downloading github.com/linkedin/goavro/v2 v2.11.1 go: downloading github.com/sirupsen/logrus v1.9.3 go: downloading github.com/pkg/errors v0.9.1 go: downloading github.com/opentracing/opentracing-go v1.2.0 go: downloading github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a go: downloading cloud.google.com/go/compute/metadata v0.3.0 go: downloading cloud.google.com/go/iam v1.1.7 go: downloading github.com/googleapis/gax-go/v2 v2.12.3 go: downloading google.golang.org/api v0.170.0 go: downloading google.golang.org/genproto v0.0.0-20240401170217-c3f982113cda go: downloading github.com/AthenZ/athenz v1.10.39 go: downloading github.com/hashicorp/go-uuid v1.0.3 go: downloading github.com/jcmturner/dnsutils/v2 v2.0.0 go: downloading google.golang.org/genproto/googleapis/api v0.0.0-20240401170217-c3f982113cda go: downloading golang.org/x/term v0.27.0 go: downloading github.com/grpc-ecosystem/grpc-gateway v1.16.0 go: downloading github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75 go: downloading go.etcd.io/bbolt v1.3.9 go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 go: downloading go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.22.0 go: downloading go.opentelemetry.io/otel v1.24.0 go: downloading go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.22.0 go: downloading go.opentelemetry.io/otel/sdk v1.24.0 go: downloading sigs.k8s.io/yaml v1.4.0 go: downloading github.com/docker/go-units v0.5.0 go: downloading github.com/tklauser/go-sysconf v0.3.12 go: downloading golang.org/x/text v0.21.0 go: downloading github.com/xdg-go/scram v1.1.2 go: downloading github.com/Masterminds/semver v1.5.0 go: downloading github.com/siddontang/go-log v0.0.0-20180807004314-8d05993dda07 go: downloading github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726 go: downloading github.com/godbus/dbus/v5 v5.0.4 go: downloading github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 go: downloading github.com/opencontainers/runtime-spec v1.0.2 go: downloading github.com/cilium/ebpf v0.4.0 go: downloading github.com/pierrec/lz4 v2.6.1+incompatible go: downloading gorm.io/gorm v1.25.11 go: downloading github.com/KyleBanks/depth v1.2.1 go: downloading github.com/go-openapi/spec v0.21.0 go: downloading golang.org/x/tools v0.28.0 go: downloading github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5 go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 go: downloading github.com/joomcode/errorx v1.0.1 go: downloading google.golang.org/genproto/googleapis/rpc v0.0.0-20240515191416-fc5f0ca64291 go: downloading github.com/gabriel-vasile/mimetype v1.4.2 go: downloading github.com/go-playground/universal-translator v0.18.1 go: downloading github.com/leodido/go-urn v1.2.4 go: downloading github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 go: downloading github.com/golang/snappy v0.0.4 go: downloading github.com/hashicorp/errwrap v1.0.0 go: downloading github.com/aws/aws-sdk-go-v2/config v1.18.30 go: downloading github.com/aws/aws-sdk-go-v2/credentials v1.13.29 go: downloading github.com/aws/aws-sdk-go-v2/service/glue v1.58.1 go: downloading github.com/jarcoal/httpmock v1.2.0 go: downloading github.com/mailru/easyjson v0.7.7 go: downloading github.com/spaolacci/murmur3 v1.1.0 go: downloading golang.org/x/mod v0.22.0 go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 go: downloading github.com/twmb/murmur3 v1.1.6 go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading github.com/cloudfoundry/gosigar v1.3.6 go: downloading golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 go: downloading go.opencensus.io v0.23.1-0.20220331163232-052120675fac go: downloading github.com/dgryski/go-farm v0.0.0-20240924180020-3414d57e47da go: downloading go.opentelemetry.io/otel/trace v1.24.0 go: downloading github.com/cockroachdb/errors v1.11.1 go: downloading github.com/cockroachdb/redact v1.1.5 go: downloading github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 go: downloading github.com/golang-jwt/jwt v3.2.2+incompatible go: downloading github.com/99designs/keyring v1.2.1 go: downloading github.com/jcmturner/aescts/v2 v2.0.0 go: downloading github.com/jcmturner/rpc/v2 v2.0.3 go: downloading github.com/gorilla/websocket v1.5.1 go: downloading go.etcd.io/etcd/raft/v3 v3.5.12 go: downloading github.com/xiang90/probing v0.0.0-20221125231312-a49e3df8f510 go: downloading github.com/jonboulle/clockwork v0.4.0 go: downloading go.opentelemetry.io/otel/metric v1.24.0 go: downloading go.opentelemetry.io/proto/otlp v1.1.0 go: downloading github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 go: downloading github.com/shopspring/decimal v1.3.0 go: downloading github.com/tklauser/numcpus v0.6.1 go: downloading github.com/xdg-go/pbkdf2 v1.0.0 go: downloading github.com/xdg-go/stringprep v1.0.4 go: downloading github.com/edwingeng/deque v0.0.0-20191220032131-8596380dee17 go: downloading github.com/jinzhu/now v1.1.5 go: downloading github.com/go-openapi/jsonpointer v0.21.0 go: downloading github.com/go-openapi/jsonreference v0.21.0 go: downloading github.com/go-playground/locales v0.14.1 go: downloading github.com/go-openapi/swag v0.23.0 go: downloading github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.6 go: downloading github.com/aws/aws-sdk-go-v2/internal/ini v1.3.37 go: downloading github.com/aws/aws-sdk-go-v2/service/sso v1.12.14 go: downloading github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.14 go: downloading github.com/aws/aws-sdk-go-v2/service/sts v1.20.1 go: downloading github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.36 go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible go: downloading github.com/go-logr/logr v1.4.1 go: downloading github.com/coocood/freecache v1.2.1 go: downloading github.com/pingcap/tipb v0.0.0-20241105053214-f91fdb81a69e go: downloading github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 go: downloading github.com/go-resty/resty/v2 v2.11.0 go: downloading github.com/ks3sdklib/aws-sdk-go v1.2.9 go: downloading github.com/otiai10/copy v1.2.0 go: downloading github.com/gorilla/mux v1.8.1 go: downloading github.com/blacktear23/go-proxyprotocol v1.0.6 go: downloading github.com/pingcap/fn v1.0.0 go: downloading github.com/carlmjohnson/flagext v0.21.0 go: downloading github.com/golang-jwt/jwt/v4 v4.5.1 go: downloading go.etcd.io/etcd/client/v2 v2.305.12 go: downloading github.com/spkg/bom v1.0.0 go: downloading github.com/xitongsys/parquet-go v1.6.3-0.20240520233950-75e935fc3e17 go: downloading github.com/tidwall/btree v1.7.0 go: downloading github.com/go-logr/stdr v1.2.2 go: downloading github.com/tiancaiamao/appdash v0.0.0-20181126055449-889f96f722a2 go: downloading github.com/sourcegraph/appdash-data v0.0.0-20151005221446-73f23eafcf67 go: downloading github.com/jinzhu/inflection v1.0.0 go: downloading github.com/qri-io/jsonschema v0.2.1 go: downloading github.com/ardielle/ardielle-go v1.5.2 go: downloading github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.30 go: downloading gopkg.in/yaml.v2 v2.4.0 go: downloading github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef go: downloading go.uber.org/mock v0.4.0 go: downloading github.com/jfcg/sorty/v2 v2.1.0 go: downloading github.com/dolthub/swiss v0.2.1 go: downloading github.com/influxdata/tdigest v0.0.1 go: downloading github.com/dvsekhvalnov/jose2go v1.5.0 go: downloading github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c go: downloading github.com/mtibben/percent v0.2.1 go: downloading github.com/opentracing/basictracer-go v1.1.0 go: downloading github.com/cheggaaa/pb/v3 v3.0.8 go: downloading github.com/dgraph-io/ristretto v0.1.1 go: downloading github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 go: downloading github.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0 go: downloading github.com/danjacques/gofslock v0.0.0-20240212154529-d899e02bfe22 go: downloading github.com/yangkeao/ldap/v3 v3.4.5-0.20230421065457-369a3bab1117 go: downloading github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.30 go: downloading github.com/lestrrat-go/jwx/v2 v2.0.21 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec go: downloading github.com/jellydator/ttlcache/v3 v3.0.1 go: downloading github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1 go: downloading github.com/google/pprof v0.0.0-20241001023024-f4c0cfd0cf1d go: downloading github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df go: downloading github.com/jedib0t/go-pretty/v6 v6.2.2 go: downloading github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 go: downloading github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f go: downloading github.com/josharian/intern v1.0.0 go: downloading github.com/dolthub/maphash v0.1.0 go: downloading github.com/pingcap/badger v1.5.1-0.20241015064302-38533b6cbf8d go: downloading github.com/vbauerster/mpb/v7 v7.5.3 go: downloading github.com/jfcg/sixb v1.3.8 go: downloading github.com/robfig/cron/v3 v3.0.1 go: downloading github.com/VividCortex/ewma v1.2.0 go: downloading github.com/mattn/go-runewidth v0.0.16 go: downloading github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 go: downloading github.com/go-asn1-ber/asn1-ber v1.5.4 go: downloading k8s.io/api v0.29.11 go: downloading github.com/emirpasic/gods v1.18.1 go: downloading github.com/apache/thrift v0.16.0 go: downloading github.com/apache/arrow/go/v12 v12.0.1 go: downloading github.com/goccy/go-reflect v1.2.0 go: downloading cloud.google.com/go/kms v1.15.8 go: downloading github.com/rivo/uniseg v0.4.7 go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible go: downloading github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b go: downloading github.com/getsentry/sentry-go v0.27.0 go: downloading github.com/kr/pretty v0.3.1 go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da go: downloading github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d go: downloading github.com/kr/text v0.2.0 go: downloading github.com/rogpeppe/go-internal v1.13.1 go: downloading github.com/golang/glog v1.2.0 go: downloading github.com/qri-io/jsonpointer v0.1.1 go: downloading github.com/kylelemons/godebug v1.1.0 go: downloading github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c go: downloading github.com/golang-jwt/jwt/v5 v5.2.1 go: downloading github.com/lestrrat-go/option v1.0.1 go: downloading github.com/lestrrat-go/blackmagic v1.0.2 go: downloading github.com/lestrrat-go/iter v1.0.2 go: downloading github.com/lestrrat-go/httprc v1.0.5 go: downloading github.com/lestrrat-go/httpcc v1.0.1 go: downloading github.com/ncw/directio v1.0.5 go: downloading github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 go: downloading github.com/klauspost/cpuid v1.3.1 go: downloading github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 go: downloading golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 go: downloading github.com/google/flatbuffers v2.0.8+incompatible go: downloading github.com/zeebo/xxh3 v1.0.2 go: downloading github.com/andybalholm/brotli v1.0.5 go: downloading github.com/klauspost/cpuid/v2 v2.2.4 go: downloading k8s.io/apimachinery v0.29.11 go: downloading k8s.io/klog/v2 v2.120.1 go: downloading github.com/google/gofuzz v1.2.0 go: downloading gopkg.in/inf.v0 v0.9.1 go: downloading sigs.k8s.io/structured-merge-diff/v4 v4.4.1 go: downloading sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd go: downloading k8s.io/utils v0.0.0-20230726121419-3b25d923346b go: downloading github.com/googleapis/enterprise-certificate-proxy v0.3.2 go: downloading go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 go: downloading github.com/google/s2a-go v0.1.7 go: downloading github.com/felixge/httpsnoop v1.0.4 go: downloading github.com/jmespath/go-jmespath v0.4.0 + '[' -f ./bin/cdc_kafka_consumer ']' + make kafka_consumer CGO_ENABLED=1 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v9.0.0-beta.1-9-g4c631d595" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2025-04-06 00:06:43" -X "github.com/pingcap/tiflow/pkg/version.GitHash=4c631d5951854ac91765b729fe3849d8e8d5ed8a" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v9.0.0-beta.1-9-g4c631d595"' -o bin/cdc_kafka_consumer ./cmd/kafka-consumer go: downloading github.com/confluentinc/confluent-kafka-go/v2 v2.3.0 go: downloading github.com/DataDog/zstd v1.5.5 + '[' -f ./bin/cdc_storage_consumer ']' + make storage_consumer CGO_ENABLED=0 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v9.0.0-beta.1-9-g4c631d595" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2025-04-06 00:09:23" -X "github.com/pingcap/tiflow/pkg/version.GitHash=4c631d5951854ac91765b729fe3849d8e8d5ed8a" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v9.0.0-beta.1-9-g4c631d595"' -o bin/cdc_storage_consumer ./cmd/storage-consumer/main.go + '[' -f ./bin/cdc.test ']' + make integration_test_build cd tools/check && GO111MODULE=on go build -mod=mod -o ../bin/failpoint-ctl github.com/pingcap/failpoint/failpoint-ctl go: downloading github.com/pingcap/failpoint v0.0.0-20210316064728-7acb0f0a3dfd go: downloading github.com/sergi/go-diff v1.1.0 CGO_ENABLED=0 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v9.0.0-beta.1-9-g4c631d595" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2025-04-06 00:09:31" -X "github.com/pingcap/tiflow/pkg/version.GitHash=4c631d5951854ac91765b729fe3849d8e8d5ed8a" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v9.0.0-beta.1-9-g4c631d595"' -o bin/cdc_storage_consumer ./cmd/storage-consumer/main.go CGO_ENABLED=1 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v9.0.0-beta.1-9-g4c631d595" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2025-04-06 00:09:31" -X "github.com/pingcap/tiflow/pkg/version.GitHash=4c631d5951854ac91765b729fe3849d8e8d5ed8a" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v9.0.0-beta.1-9-g4c631d595"' -o bin/cdc_kafka_consumer ./cmd/kafka-consumer CGO_ENABLED=0 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v9.0.0-beta.1-9-g4c631d595" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2025-04-06 00:09:31" -X "github.com/pingcap/tiflow/pkg/version.GitHash=4c631d5951854ac91765b729fe3849d8e8d5ed8a" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v9.0.0-beta.1-9-g4c631d595"' -o bin/cdc_pulsar_consumer ./cmd/pulsar-consumer/main.go CGO_ENABLED=0 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v9.0.0-beta.1-9-g4c631d595" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2025-04-06 00:09:31" -X "github.com/pingcap/tiflow/pkg/version.GitHash=4c631d5951854ac91765b729fe3849d8e8d5ed8a" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v9.0.0-beta.1-9-g4c631d595"' -o bin/oauth2-server ./cmd/oauth2-server/main.go go: downloading github.com/go-oauth2/oauth2/v4 v4.5.2 go: downloading github.com/tidwall/buntdb v1.3.0 go: downloading github.com/tidwall/rtred v0.1.2 go: downloading github.com/tidwall/match v1.1.1 go: downloading github.com/tidwall/grect v0.1.4 go: downloading github.com/tidwall/gjson v1.14.3 go: downloading github.com/tidwall/tinyqueue v0.1.1 go: downloading github.com/tidwall/pretty v1.2.0 CGO_ENABLED=0 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v9.0.0-beta.1-9-g4c631d595" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2025-04-06 00:09:31" -X "github.com/pingcap/tiflow/pkg/version.GitHash=4c631d5951854ac91765b729fe3849d8e8d5ed8a" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v9.0.0-beta.1-9-g4c631d595"' -o bin/sync_diff_inspector ./sync_diff_inspector go: downloading github.com/olekukonko/tablewriter v0.0.5 $(echo $(for p in $(go list ./... | grep -vE 'vendor|proto|tiflow/tests|integration|testing_utils|pb|pbmock|tiflow/bin'); do echo ${p#"github.com/pingcap/tiflow/"}|grep -v "github.com/pingcap/tiflow"; done) | xargs tools/bin/failpoint-ctl enable >/dev/null) go: downloading github.com/PingCAP-QE/go-sqlsmith v0.0.0-20241224111350-ad2e4f976c7c go: downloading github.com/chzyer/readline v1.5.1 go: downloading github.com/deepmap/oapi-codegen v1.9.0 go: downloading github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 go: downloading github.com/gogo/gateway v1.1.0 go: downloading github.com/getkin/kin-openapi v0.80.0 go: downloading github.com/shurcooL/httpgzip v0.0.0-20190720172056-320755c1c1b0 go: downloading github.com/syndtr/goleveldb v1.0.1-0.20210305035536-64b5b1c73954 go: downloading github.com/mattn/go-shellwords v1.0.12 go: downloading github.com/go-ozzo/ozzo-validation/v4 v4.3.0 go: downloading github.com/ngaut/log v0.0.0-20210830112240-0124ec040aeb go: downloading go.uber.org/dig v1.13.0 go: downloading github.com/glebarez/go-sqlite v1.21.2 go: downloading go.uber.org/ratelimit v0.2.0 go: downloading github.com/VividCortex/mysqlerr v1.0.0 go: downloading go.uber.org/goleak v1.3.0 go: downloading upper.io/db.v3 v3.7.1+incompatible go: downloading github.com/glebarez/sqlite v1.7.0 go: downloading github.com/jmoiron/sqlx v1.3.3 go: downloading github.com/integralist/go-findroot v0.0.0-20160518114804-ac90681525dc go: downloading gorm.io/driver/mysql v1.5.7 go: downloading github.com/bradleyjkemp/grpc-tools v0.2.5 go: downloading github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 go: downloading modernc.org/sqlite v1.27.0 go: downloading modernc.org/libc v1.37.1 go: downloading github.com/improbable-eng/grpc-web v0.12.0 go: downloading github.com/ghodss/yaml v1.0.0 go: downloading github.com/rs/cors v1.7.0 go: downloading github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f go: downloading modernc.org/memory v1.7.2 go: downloading modernc.org/mathutil v1.6.0 CGO_ENABLED=1 GO111MODULE=on go test -p 3 --race --tags=intest -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v9.0.0-beta.1-9-g4c631d595" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2025-04-06 00:09:31" -X "github.com/pingcap/tiflow/pkg/version.GitHash=4c631d5951854ac91765b729fe3849d8e8d5ed8a" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v9.0.0-beta.1-9-g4c631d595"' -c -cover -covermode=atomic \ -coverpkg=github.com/pingcap/tiflow/... \ -o bin/cdc.test github.com/pingcap/tiflow/cmd/cdc \ || { $(echo $(for p in $(go list ./... | grep -vE 'vendor|proto|tiflow/tests|integration|testing_utils|pb|pbmock|tiflow/bin'); do echo ${p#"github.com/pingcap/tiflow/"}|grep -v "github.com/pingcap/tiflow"; done) | xargs tools/bin/failpoint-ctl disable >/dev/null); exit 1; } CGO_ENABLED=0 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v9.0.0-beta.1-9-g4c631d595" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2025-04-06 00:09:31" -X "github.com/pingcap/tiflow/pkg/version.GitHash=4c631d5951854ac91765b729fe3849d8e8d5ed8a" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v9.0.0-beta.1-9-g4c631d595"' -o bin/cdc ./cmd/cdc/main.go \ || { $(echo $(for p in $(go list ./... | grep -vE 'vendor|proto|tiflow/tests|integration|testing_utils|pb|pbmock|tiflow/bin'); do echo ${p#"github.com/pingcap/tiflow/"}|grep -v "github.com/pingcap/tiflow"; done) | xargs tools/bin/failpoint-ctl disable >/dev/null); exit 1; } $(echo $(for p in $(go list ./... | grep -vE 'vendor|proto|tiflow/tests|integration|testing_utils|pb|pbmock|tiflow/bin'); do echo ${p#"github.com/pingcap/tiflow/"}|grep -v "github.com/pingcap/tiflow"; done) | xargs tools/bin/failpoint-ctl disable >/dev/null) + ls -alh ./bin total 1.4G drwxr-sr-x 2 jenkins jenkins 4.0K Apr 6 08:15 . drwxr-sr-x 19 jenkins jenkins 4.0K Apr 6 08:03 .. -rwxr-xr-x 1 jenkins jenkins 231M Apr 6 08:15 cdc -rwxr-xr-x 1 jenkins jenkins 363M Apr 6 08:14 cdc.test -rwxr-xr-x 1 jenkins jenkins 268M Apr 6 08:10 cdc_kafka_consumer -rwxr-xr-x 1 jenkins jenkins 200M Apr 6 08:10 cdc_pulsar_consumer -rwxr-xr-x 1 jenkins jenkins 199M Apr 6 08:09 cdc_storage_consumer -rwxr-xr-x 1 jenkins jenkins 13M Apr 6 08:10 oauth2-server -rwxr-xr-x 1 jenkins jenkins 145M Apr 6 08:10 sync_diff_inspector + ./bin/cdc version Release Version: v9.0.0-beta.1-9-g4c631d595 Git Commit Hash: 4c631d5951854ac91765b729fe3849d8e8d5ed8a Git Branch: HEAD UTC Build Time: 2025-04-06 00:09:31 Go Version: go1.23.4 Failpoint Build: true [Pipeline] cache Cache not restored (no such key found) [Pipeline] { [Pipeline] sh + cp -r ../third_party_download/bin/bin ../third_party_download/bin/etc ../third_party_download/bin/etcdctl ../third_party_download/bin/go-ycsb ../third_party_download/bin/jq ../third_party_download/bin/lib ../third_party_download/bin/libc++.so.1 ../third_party_download/bin/libc++.so.1.0 ../third_party_download/bin/libc++abi.so.1 ../third_party_download/bin/libc++abi.so.1.0 ../third_party_download/bin/libgmssl.so ../third_party_download/bin/libgmssl.so.3 ../third_party_download/bin/libgmssl.so.3.0 ../third_party_download/bin/libtiflash_proxy.so ../third_party_download/bin/minio ../third_party_download/bin/pd-api-bench ../third_party_download/bin/pd-ctl ../third_party_download/bin/pd-heartbeat-bench ../third_party_download/bin/pd-recover ../third_party_download/bin/pd-region-bench ../third_party_download/bin/pd-server ../third_party_download/bin/pd-tso-bench ../third_party_download/bin/pd-ut ../third_party_download/bin/regions-dump ../third_party_download/bin/share ../third_party_download/bin/stores-dump ../third_party_download/bin/tidb-server ../third_party_download/bin/tiflash ../third_party_download/bin/tikv-server ../third_party_download/bin/xprog ./bin/ + ls -alh ./bin total 3.3G drwxr-sr-x 6 jenkins jenkins 4.0K Apr 6 08:15 . drwxr-sr-x 19 jenkins jenkins 4.0K Apr 6 08:03 .. drwxr-sr-x 2 jenkins jenkins 4.0K Apr 6 08:15 bin -rwxr-xr-x 1 jenkins jenkins 231M Apr 6 08:15 cdc -rwxr-xr-x 1 jenkins jenkins 363M Apr 6 08:14 cdc.test -rwxr-xr-x 1 jenkins jenkins 268M Apr 6 08:10 cdc_kafka_consumer -rwxr-xr-x 1 jenkins jenkins 200M Apr 6 08:10 cdc_pulsar_consumer -rwxr-xr-x 1 jenkins jenkins 199M Apr 6 08:09 cdc_storage_consumer drwxr-sr-x 4 jenkins jenkins 4.0K Apr 6 08:15 etc -rwxr-xr-x 1 jenkins jenkins 17M Apr 6 08:15 etcdctl -rwxr-xr-x 1 jenkins jenkins 44M Apr 6 08:15 go-ycsb -rwxr-xr-x 1 jenkins jenkins 3.8M Apr 6 08:15 jq drwxr-sr-x 3 jenkins jenkins 4.0K Apr 6 08:15 lib lrwxrwxrwx 1 jenkins jenkins 13 Apr 6 08:15 libc++.so.1 -> libc++.so.1.0 -rwxr-xr-x 1 jenkins jenkins 1.2M Apr 6 08:15 libc++.so.1.0 lrwxrwxrwx 1 jenkins jenkins 16 Apr 6 08:15 libc++abi.so.1 -> libc++abi.so.1.0 -rwxr-xr-x 1 jenkins jenkins 353K Apr 6 08:15 libc++abi.so.1.0 lrwxrwxrwx 1 jenkins jenkins 13 Apr 6 08:15 libgmssl.so -> libgmssl.so.3 lrwxrwxrwx 1 jenkins jenkins 15 Apr 6 08:15 libgmssl.so.3 -> libgmssl.so.3.0 -rwxr-xr-x 1 jenkins jenkins 2.2M Apr 6 08:15 libgmssl.so.3.0 -rwxr-xr-x 1 jenkins jenkins 346M Apr 6 08:15 libtiflash_proxy.so -rwxr-xr-x 1 jenkins jenkins 50M Apr 6 08:15 minio -rwxr-xr-x 1 jenkins jenkins 13M Apr 6 08:10 oauth2-server -rwxr-xr-x 1 jenkins jenkins 41M Apr 6 08:15 pd-api-bench -rwxr-xr-x 1 jenkins jenkins 44M Apr 6 08:15 pd-ctl -rwxr-xr-x 1 jenkins jenkins 40M Apr 6 08:15 pd-heartbeat-bench -rwxr-xr-x 1 jenkins jenkins 35M Apr 6 08:15 pd-recover -rwxr-xr-x 1 jenkins jenkins 37M Apr 6 08:15 pd-region-bench -rwxr-xr-x 1 jenkins jenkins 115M Apr 6 08:15 pd-server -rwxr-xr-x 1 jenkins jenkins 37M Apr 6 08:15 pd-tso-bench -rwxr-xr-x 1 jenkins jenkins 15M Apr 6 08:15 pd-ut -rwxr-xr-x 1 jenkins jenkins 35M Apr 6 08:15 regions-dump drwxr-sr-x 4 jenkins jenkins 4.0K Apr 6 08:15 share -rwxr-xr-x 1 jenkins jenkins 35M Apr 6 08:15 stores-dump -rwxr-xr-x 1 jenkins jenkins 145M Apr 6 08:10 sync_diff_inspector -rwxr-xr-x 1 jenkins jenkins 241M Apr 6 08:15 tidb-server -rwxr-xr-x 1 jenkins jenkins 443M Apr 6 08:15 tiflash -rwxr-xr-x 1 jenkins jenkins 373M Apr 6 08:15 tikv-server -rwxr-xr-x 1 jenkins jenkins 2.3M Apr 6 08:15 xprog [Pipeline] } Cache saved successfully (ws/jenkins-pingcap-tiflow-periodics_cdc_integration_mysql_test-234/tiflow-cdc) 4028702720 bytes in 92.91 secs (43363136 bytes/sec) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // stage [Pipeline] stage [Pipeline] { (Tests) [Pipeline] parallel [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G00') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G01') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G02') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G03') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G04') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G05') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G06') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G07') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G08') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G09') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G10') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G11') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G12') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G13') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G14') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G15') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G16') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G17') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G18') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G19') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G20') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G21') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G00') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G01') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G02') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G03') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G04') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G05') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G06') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G07') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G08') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G09') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G10') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G11') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G12') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G13') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G14') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G15') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G16') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G17') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G18') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G19') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G20') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G21') [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted Obtained pipelines/pingcap/tiflow/latest/pod-periodics_cdc_integration_mysql_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Obtained pipelines/pingcap/tiflow/latest/pod-periodics_cdc_integration_mysql_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-periodics-cdc-integration-mysql-test-234-9-92vm1 Obtained pipelines/pingcap/tiflow/latest/pod-periodics_cdc_integration_mysql_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Obtained pipelines/pingcap/tiflow/latest/pod-periodics_cdc_integration_mysql_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Obtained pipelines/pingcap/tiflow/latest/pod-periodics_cdc_integration_mysql_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-periodics-cdc-integration-mysql-test-234-k-bx9s4 Agent pingcap-tiflow-periodics-cdc-integration-mysql-test-234-9-92vm1 is provisioned from template pingcap_tiflow_periodics_cdc_integration_mysql_test_234-95xmf-8srd7 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/" runUrl: "job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "0243c660601548c232bf3b633d23f72b36d0cb60" jenkins/label: "pingcap_tiflow_periodics_cdc_integration_mysql_test_234-95xmf" name: "pingcap-tiflow-periodics-cdc-integration-mysql-test-234-9-92vm1" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.23:latest" name: "golang" resources: limits: memory: "32Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-periodics-cdc-integration-mysql-test-234-9-92vm1" - name: "JENKINS_NAME" value: "pingcap-tiflow-periodics-cdc-integration-mysql-test-234-9-92vm1" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-periodics-cdc-integration-mysql-test-234-9-92vm1 in /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test [Pipeline] { [Pipeline] checkout The recommended git tool is: git Obtained pipelines/pingcap/tiflow/latest/pod-periodics_cdc_integration_mysql_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-periodics-cdc-integration-mysql-test-234-c-m09xf No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@3ae0066f; decorates RemoteLauncher[hudson.remoting.Channel@1ccbda82:JNLP4-connect connection from 10.233.77.139/10.233.77.139:33570] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Obtained pipelines/pingcap/tiflow/latest/pod-periodics_cdc_integration_mysql_test.yaml from git https://github.com/PingCAP-QE/ci.git Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 [Pipeline] podTemplate [Pipeline] { [Pipeline] node Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-periodics-cdc-integration-mysql-test-234-v-hm9ds Obtained pipelines/pingcap/tiflow/latest/pod-periodics_cdc_integration_mysql_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-periodics-cdc-integration-mysql-test-234-m-cm2pq Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-periodics-cdc-integration-mysql-test-234-v-nsng0 Obtained pipelines/pingcap/tiflow/latest/pod-periodics_cdc_integration_mysql_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Avoid second fetch Checking out Revision b2a59508c11c86ba701c4af25fc536a9475b82c6 (origin/main) > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f b2a59508c11c86ba701c4af25fc536a9475b82c6 # timeout=10 Commit message: "feat(prow-jobs/pingcap-qe/ci): add periodic job to crawl pull requests and issues (#3449)" [Pipeline] withEnv [Pipeline] { Agent pingcap-tiflow-periodics-cdc-integration-mysql-test-234-k-bx9s4 is provisioned from template pingcap_tiflow_periodics_cdc_integration_mysql_test_234-krkrh-p2ks1 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/" runUrl: "job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "1d5826cd7fb808c9e7f86ef192ad6ccf17b0f9bb" jenkins/label: "pingcap_tiflow_periodics_cdc_integration_mysql_test_234-krkrh" name: "pingcap-tiflow-periodics-cdc-integration-mysql-test-234-k-bx9s4" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.23:latest" name: "golang" resources: limits: memory: "32Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-periodics-cdc-integration-mysql-test-234-k-bx9s4" - name: "JENKINS_NAME" value: "pingcap-tiflow-periodics-cdc-integration-mysql-test-234-k-bx9s4" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-periodics-cdc-integration-mysql-test-234-k-bx9s4 in /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test [Pipeline] container [Pipeline] { [Pipeline] { [Pipeline] checkout [Pipeline] stage The recommended git tool is: git [Pipeline] { (Test) [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow [Pipeline] { [Pipeline] cache Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-periodics-cdc-integration-mysql-test-234-6-zhp59 Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-periodics-cdc-integration-mysql-test-234-8-31x9h Obtained pipelines/pingcap/tiflow/latest/pod-periodics_cdc_integration_mysql_test.yaml from git https://github.com/PingCAP-QE/ci.git Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-periodics-cdc-integration-mysql-test-234-d-7dnbd No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@5adf6ebe; decorates RemoteLauncher[hudson.remoting.Channel@18eb6368:JNLP4-connect connection from 10.233.72.215/10.233.72.215:34944] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Obtained pipelines/pingcap/tiflow/latest/pod-periodics_cdc_integration_mysql_test.yaml from git https://github.com/PingCAP-QE/ci.git Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Obtained pipelines/pingcap/tiflow/latest/pod-periodics_cdc_integration_mysql_test.yaml from git https://github.com/PingCAP-QE/ci.git Agent pingcap-tiflow-periodics-cdc-integration-mysql-test-234-m-cm2pq is provisioned from template pingcap_tiflow_periodics_cdc_integration_mysql_test_234-m9519-72g7b --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/" runUrl: "job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "cee696a8d987f275b1830e80504777dc0b510ae1" jenkins/label: "pingcap_tiflow_periodics_cdc_integration_mysql_test_234-m9519" name: "pingcap-tiflow-periodics-cdc-integration-mysql-test-234-m-cm2pq" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.23:latest" name: "golang" resources: limits: memory: "32Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-periodics-cdc-integration-mysql-test-234-m-cm2pq" - name: "JENKINS_NAME" value: "pingcap-tiflow-periodics-cdc-integration-mysql-test-234-m-cm2pq" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-periodics-cdc-integration-mysql-test-234-m-cm2pq in /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test Agent pingcap-tiflow-periodics-cdc-integration-mysql-test-234-c-m09xf is provisioned from template pingcap_tiflow_periodics_cdc_integration_mysql_test_234-cf22g-px2mb --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/" runUrl: "job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "66d8d8803da9394be6ef328489aecfe70999de42" jenkins/label: "pingcap_tiflow_periodics_cdc_integration_mysql_test_234-cf22g" name: "pingcap-tiflow-periodics-cdc-integration-mysql-test-234-c-m09xf" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.23:latest" name: "golang" resources: limits: memory: "32Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-periodics-cdc-integration-mysql-test-234-c-m09xf" - name: "JENKINS_NAME" value: "pingcap-tiflow-periodics-cdc-integration-mysql-test-234-c-m09xf" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-periodics-cdc-integration-mysql-test-234-c-m09xf in /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test Obtained pipelines/pingcap/tiflow/latest/pod-periodics_cdc_integration_mysql_test.yaml from git https://github.com/PingCAP-QE/ci.git > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 Avoid second fetch Checking out Revision b2a59508c11c86ba701c4af25fc536a9475b82c6 (origin/main) Commit message: "feat(prow-jobs/pingcap-qe/ci): add periodic job to crawl pull requests and issues (#3449)" Obtained pipelines/pingcap/tiflow/latest/pod-periodics_cdc_integration_mysql_test.yaml from git https://github.com/PingCAP-QE/ci.git Agent pingcap-tiflow-periodics-cdc-integration-mysql-test-234-8-31x9h is provisioned from template pingcap_tiflow_periodics_cdc_integration_mysql_test_234-84pgq-pjw6v --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/" runUrl: "job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "38c37256cc8951be4219c1a455e648f1458970fb" jenkins/label: "pingcap_tiflow_periodics_cdc_integration_mysql_test_234-84pgq" name: "pingcap-tiflow-periodics-cdc-integration-mysql-test-234-8-31x9h" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.23:latest" name: "golang" resources: limits: memory: "32Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-periodics-cdc-integration-mysql-test-234-8-31x9h" - name: "JENKINS_NAME" value: "pingcap-tiflow-periodics-cdc-integration-mysql-test-234-8-31x9h" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Agent pingcap-tiflow-periodics-cdc-integration-mysql-test-234-v-hm9ds is provisioned from template pingcap_tiflow_periodics_cdc_integration_mysql_test_234-vf84t-gkqmb --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/" runUrl: "job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "33440d1182e01d61c150c6d70d8dc896a43e7909" jenkins/label: "pingcap_tiflow_periodics_cdc_integration_mysql_test_234-vf84t" name: "pingcap-tiflow-periodics-cdc-integration-mysql-test-234-v-hm9ds" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.23:latest" name: "golang" resources: limits: memory: "32Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-periodics-cdc-integration-mysql-test-234-v-hm9ds" - name: "JENKINS_NAME" value: "pingcap-tiflow-periodics-cdc-integration-mysql-test-234-v-hm9ds" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-periodics-cdc-integration-mysql-test-234-8-31x9h in /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test Running on pingcap-tiflow-periodics-cdc-integration-mysql-test-234-v-hm9ds in /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f b2a59508c11c86ba701c4af25fc536a9475b82c6 # timeout=10 Agent pingcap-tiflow-periodics-cdc-integration-mysql-test-234-v-nsng0 is provisioned from template pingcap_tiflow_periodics_cdc_integration_mysql_test_234-vvls1-71zf2 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/" runUrl: "job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "19ba93a8e687432a2ea6a05bfeb22f866cd329d3" jenkins/label: "pingcap_tiflow_periodics_cdc_integration_mysql_test_234-vvls1" name: "pingcap-tiflow-periodics-cdc-integration-mysql-test-234-v-nsng0" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.23:latest" name: "golang" resources: limits: memory: "32Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-periodics-cdc-integration-mysql-test-234-v-nsng0" - name: "JENKINS_NAME" value: "pingcap-tiflow-periodics-cdc-integration-mysql-test-234-v-nsng0" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Agent pingcap-tiflow-periodics-cdc-integration-mysql-test-234-6-zhp59 is provisioned from template pingcap_tiflow_periodics_cdc_integration_mysql_test_234-693tt-bh1pk --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/" runUrl: "job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "1636467d362dc21db2e8bc671fd39c9122a4b980" jenkins/label: "pingcap_tiflow_periodics_cdc_integration_mysql_test_234-693tt" name: "pingcap-tiflow-periodics-cdc-integration-mysql-test-234-6-zhp59" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.23:latest" name: "golang" resources: limits: memory: "32Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-periodics-cdc-integration-mysql-test-234-6-zhp59" - name: "JENKINS_NAME" value: "pingcap-tiflow-periodics-cdc-integration-mysql-test-234-6-zhp59" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-periodics-cdc-integration-mysql-test-234-v-nsng0 in /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test Running on pingcap-tiflow-periodics-cdc-integration-mysql-test-234-6-zhp59 in /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test Obtained pipelines/pingcap/tiflow/latest/pod-periodics_cdc_integration_mysql_test.yaml from git https://github.com/PingCAP-QE/ci.git Obtained pipelines/pingcap/tiflow/latest/pod-periodics_cdc_integration_mysql_test.yaml from git https://github.com/PingCAP-QE/ci.git Agent pingcap-tiflow-periodics-cdc-integration-mysql-test-234-d-7dnbd is provisioned from template pingcap_tiflow_periodics_cdc_integration_mysql_test_234-dbhbr-l0cc5 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/" runUrl: "job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "efa5885bd0a419fdc3018f64b1dab35b70592fab" jenkins/label: "pingcap_tiflow_periodics_cdc_integration_mysql_test_234-dbhbr" name: "pingcap-tiflow-periodics-cdc-integration-mysql-test-234-d-7dnbd" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.23:latest" name: "golang" resources: limits: memory: "32Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-periodics-cdc-integration-mysql-test-234-d-7dnbd" - name: "JENKINS_NAME" value: "pingcap-tiflow-periodics-cdc-integration-mysql-test-234-d-7dnbd" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-periodics-cdc-integration-mysql-test-234-d-7dnbd in /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test Obtained pipelines/pingcap/tiflow/latest/pod-periodics_cdc_integration_mysql_test.yaml from git https://github.com/PingCAP-QE/ci.git Obtained pipelines/pingcap/tiflow/latest/pod-periodics_cdc_integration_mysql_test.yaml from git https://github.com/PingCAP-QE/ci.git Obtained pipelines/pingcap/tiflow/latest/pod-periodics_cdc_integration_mysql_test.yaml from git https://github.com/PingCAP-QE/ci.git Cache restored successfully (ws/jenkins-pingcap-tiflow-periodics_cdc_integration_mysql_test-234/tiflow-cdc) 4028702720 bytes in 6.98 secs (576961925 bytes/sec) [Pipeline] { Obtained pipelines/pingcap/tiflow/latest/pod-periodics_cdc_integration_mysql_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] timeout Timeout set to expire in 45 min [Pipeline] { [Pipeline] sh [Pipeline] node [Pipeline] node [Pipeline] node [Pipeline] node Obtained pipelines/pingcap/tiflow/latest/pod-periodics_cdc_integration_mysql_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] node [Pipeline] node + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G00 Run cases: down_db_require_secure_transport bdr_mode capture_suicide_while_balance_table syncpoint syncpoint_check_ts server_config_compatibility changefeed_dup_error_restart safe_mode kafka_big_messages kafka_compression kafka_messages kafka_sink_error_resume mq_sink_lost_callback mq_sink_dispatcher kafka_column_selector kafka_column_selector_avro debezium lossy_ddl storage_csv_update csv_storage_update_pk_clustered csv_storage_update_pk_nonclustered PROW_JOB_ID=1ff7fcb7-127a-11f0-bebb-96e16cfa81f4 JENKINS_NODE_COOKIE=cd700e81-fd73-428b-9f61-c8f0dbc8bb39 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/ GOLANG_VERSION=1.23.3 HOSTNAME=pingcap-tiflow-periodics-cdc-integration-mysql-test-234-9-92vm1 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-periodics_cdc_integration_mysql_test-234 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 JOB_SPEC={"type":"periodic","job":"pingcap/tiflow/periodics_cdc_integration_mysql_test","buildid":"1908671202193313792","prowjobid":"1ff7fcb7-127a-11f0-bebb-96e16cfa81f4","extra_refs":[{"org":"pingcap","repo":"tiflow","base_ref":"master"}]} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ JOB_NAME=pingcap/tiflow/periodics_cdc_integration_mysql_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#234 TEST_GROUP=G00 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1908671202193313792 GOLANG_DOWNLOAD_SHA256=a0afb9744c00648bafb1b90b4aba5bdb86f424f02f9275399ce0c20b93a2c3a8 JOB_BASE_NAME=periodics_cdc_integration_mysql_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_periodics_cdc_integration_mysql_test_234-95xmf GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-8f9954e717ae06be5a19c339deaa805ddd7d685252d85c8682580d893f46201d NODE_LABELS=pingcap_tiflow_periodics_cdc_integration_mysql_test_234-95xmf pingcap-tiflow-periodics-cdc-integration-mysql-test-234-9-92vm1 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-periodics-cdc-integration-mysql-test-234-9-92vm1 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/display/redirect BUILD_NUMBER=234 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.23.3.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/down_db_require_secure_transport/run.sh using Sink-Type: mysql... <<================= [Pipeline] node [Pipeline] node [Pipeline] node [Pipeline] node [Pipeline] { [Pipeline] { The 1 times to try to start tidb cluster... [Pipeline] withEnv [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] { [Pipeline] checkout [Pipeline] checkout The recommended git tool is: git The recommended git tool is: git [Pipeline] podTemplate [Pipeline] { [Pipeline] checkout [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout [Pipeline] checkout The recommended git tool is: git The recommended git tool is: git Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-periodics-cdc-integration-mysql-test-234-c-g3td9 [Pipeline] checkout Obtained pipelines/pingcap/tiflow/latest/pod-periodics_cdc_integration_mysql_test.yaml from git https://github.com/PingCAP-QE/ci.git The recommended git tool is: git [Pipeline] node Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-periodics-cdc-integration-mysql-test-234-j-shst4 The recommended git tool is: git Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-periodics-cdc-integration-mysql-test-234-j-vgmw9 Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-periodics-cdc-integration-mysql-test-234-4-tjmd8 [Pipeline] podTemplate [Pipeline] { Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-periodics-cdc-integration-mysql-test-234-x-4tfw6 [Pipeline] stage [Pipeline] { (Test) [Pipeline] node Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-periodics-cdc-integration-mysql-test-234-m-66zpf Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-periodics-cdc-integration-mysql-test-234-x-hbn5c [Pipeline] podTemplate [Pipeline] { Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-periodics-cdc-integration-mysql-test-234-2-dwpgc [Pipeline] node [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-periodics-cdc-integration-mysql-test-234-j-x7t22 [Pipeline] { [Pipeline] cache Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-periodics-cdc-integration-mysql-test-234-p-5tjc6 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@1ee4636b; decorates RemoteLauncher[hudson.remoting.Channel@803693a:JNLP4-connect connection from 10.233.107.135/10.233.107.135:53432] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@330c31e1; decorates RemoteLauncher[hudson.remoting.Channel@6a0cca19:JNLP4-connect connection from 10.233.77.233/10.233.77.233:60614] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@6a06b14f; decorates RemoteLauncher[hudson.remoting.Channel@20da8d75:JNLP4-connect connection from 10.233.78.188/10.233.78.188:45728] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@2c93340c; decorates RemoteLauncher[hudson.remoting.Channel@7b9248e3:JNLP4-connect connection from 10.233.123.118/10.233.123.118:43026] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@61ced183; decorates RemoteLauncher[hudson.remoting.Channel@52be141f:JNLP4-connect connection from 10.233.107.91/10.233.107.91:37192] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@55dbb06b; decorates RemoteLauncher[hudson.remoting.Channel@1a00f73a:JNLP4-connect connection from 10.233.75.94/10.233.75.94:54302] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@6241de32; decorates RemoteLauncher[hudson.remoting.Channel@37852a6f:JNLP4-connect connection from 10.233.72.62/10.233.72.62:34922] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-periodics-cdc-integration-mysql-test-234-z-4sl2f Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-periodics-cdc-integration-mysql-test-234-x-pbmg2 Avoid second fetch Checking out Revision b2a59508c11c86ba701c4af25fc536a9475b82c6 (origin/main) Avoid second fetch Checking out Revision b2a59508c11c86ba701c4af25fc536a9475b82c6 (origin/main) > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f b2a59508c11c86ba701c4af25fc536a9475b82c6 # timeout=10 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f b2a59508c11c86ba701c4af25fc536a9475b82c6 # timeout=10 Commit message: "feat(prow-jobs/pingcap-qe/ci): add periodic job to crawl pull requests and issues (#3449)" Avoid second fetch Checking out Revision b2a59508c11c86ba701c4af25fc536a9475b82c6 (origin/main) Avoid second fetch Agent pingcap-tiflow-periodics-cdc-integration-mysql-test-234-m-66zpf is provisioned from template pingcap_tiflow_periodics_cdc_integration_mysql_test_234-mqwq4-j5j6c --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/" runUrl: "job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "216d7eede5a57fb20aab5b907763613a3cdeb53f" jenkins/label: "pingcap_tiflow_periodics_cdc_integration_mysql_test_234-mqwq4" name: "pingcap-tiflow-periodics-cdc-integration-mysql-test-234-m-66zpf" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.23:latest" name: "golang" resources: limits: memory: "32Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-periodics-cdc-integration-mysql-test-234-m-66zpf" - name: "JENKINS_NAME" value: "pingcap-tiflow-periodics-cdc-integration-mysql-test-234-m-66zpf" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Checking out Revision b2a59508c11c86ba701c4af25fc536a9475b82c6 (origin/main) Running on pingcap-tiflow-periodics-cdc-integration-mysql-test-234-m-66zpf in /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test Commit message: "feat(prow-jobs/pingcap-qe/ci): add periodic job to crawl pull requests and issues (#3449)" Agent pingcap-tiflow-periodics-cdc-integration-mysql-test-234-j-shst4 is provisioned from template pingcap_tiflow_periodics_cdc_integration_mysql_test_234-jh1tm-vdkp4 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/" runUrl: "job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "ed9f67c74b23b665e33e47b8a7c49832866155ad" jenkins/label: "pingcap_tiflow_periodics_cdc_integration_mysql_test_234-jh1tm" name: "pingcap-tiflow-periodics-cdc-integration-mysql-test-234-j-shst4" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.23:latest" name: "golang" resources: limits: memory: "32Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-periodics-cdc-integration-mysql-test-234-j-shst4" - name: "JENKINS_NAME" value: "pingcap-tiflow-periodics-cdc-integration-mysql-test-234-j-shst4" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 Running on pingcap-tiflow-periodics-cdc-integration-mysql-test-234-j-shst4 in /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test Avoid second fetch Checking out Revision b2a59508c11c86ba701c4af25fc536a9475b82c6 (origin/main) > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f b2a59508c11c86ba701c4af25fc536a9475b82c6 # timeout=10 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 Commit message: "feat(prow-jobs/pingcap-qe/ci): add periodic job to crawl pull requests and issues (#3449)" > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f b2a59508c11c86ba701c4af25fc536a9475b82c6 # timeout=10 Commit message: "feat(prow-jobs/pingcap-qe/ci): add periodic job to crawl pull requests and issues (#3449)" Agent pingcap-tiflow-periodics-cdc-integration-mysql-test-234-4-tjmd8 is provisioned from template pingcap_tiflow_periodics_cdc_integration_mysql_test_234-46hg9-bfw3h --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/" runUrl: "job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "28c8dfdba464884bc23b8722e0f2aadd9d096ca4" jenkins/label: "pingcap_tiflow_periodics_cdc_integration_mysql_test_234-46hg9" name: "pingcap-tiflow-periodics-cdc-integration-mysql-test-234-4-tjmd8" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.23:latest" name: "golang" resources: limits: memory: "32Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-periodics-cdc-integration-mysql-test-234-4-tjmd8" - name: "JENKINS_NAME" value: "pingcap-tiflow-periodics-cdc-integration-mysql-test-234-4-tjmd8" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-periodics-cdc-integration-mysql-test-234-4-tjmd8 in /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 Avoid second fetch Checking out Revision b2a59508c11c86ba701c4af25fc536a9475b82c6 (origin/main) Commit message: "feat(prow-jobs/pingcap-qe/ci): add periodic job to crawl pull requests and issues (#3449)" Avoid second fetch Checking out Revision b2a59508c11c86ba701c4af25fc536a9475b82c6 (origin/main) Agent pingcap-tiflow-periodics-cdc-integration-mysql-test-234-c-g3td9 is provisioned from template pingcap_tiflow_periodics_cdc_integration_mysql_test_234-cqxrw-tz4t6 Agent pingcap-tiflow-periodics-cdc-integration-mysql-test-234-p-5tjc6 is provisioned from template pingcap_tiflow_periodics_cdc_integration_mysql_test_234-p2wz0-p5vr6 --- --- apiVersion: "v1" kind: "Pod" metadata: annotations: apiVersion: "v1" buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/" kind: "Pod" metadata: annotations: runUrl: "job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "a4c020ba2a27fd56c1a2ee086565a38b55aab244" buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/" runUrl: "job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/" labels: jenkins/label: "pingcap_tiflow_periodics_cdc_integration_mysql_test_234-p2wz0" name: "pingcap-tiflow-periodics-cdc-integration-mysql-test-234-p-5tjc6" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "87b6819d3476b50cc11468d97863f470e01bfe06" requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" jenkins/label: "pingcap_tiflow_periodics_cdc_integration_mysql_test_234-cqxrw" name: "pingcap-tiflow-periodics-cdc-integration-mysql-test-234-c-g3td9" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.23:latest" name: "golang" resources: limits: memory: "32Gi" cpu: "6" requests: memory: "12Gi" operator: "In" values: - "amd64" containers: cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: - image: "hub.pingcap.net/jenkins/centos7_golang-1.23:latest" name: "golang" resources: limits: memory: "32Gi" cpu: "6" memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true value: "pingcap-tiflow-periodics-cdc-integration-mysql-test-234-c-g3td9" - name: "JENKINS_NAME" value: "pingcap-tiflow-periodics-cdc-integration-mysql-test-234-c-g3td9" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-periodics-cdc-integration-mysql-test-234-p-5tjc6" - name: "JENKINS_NAME" value: "pingcap-tiflow-periodics-cdc-integration-mysql-test-234-p-5tjc6" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-periodics-cdc-integration-mysql-test-234-p-5tjc6 in /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test Running on pingcap-tiflow-periodics-cdc-integration-mysql-test-234-c-g3td9 in /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test Commit message: "feat(prow-jobs/pingcap-qe/ci): add periodic job to crawl pull requests and issues (#3449)" Agent pingcap-tiflow-periodics-cdc-integration-mysql-test-234-j-vgmw9 is provisioned from template pingcap_tiflow_periodics_cdc_integration_mysql_test_234-jpjd1-rftp0 Commit message: "feat(prow-jobs/pingcap-qe/ci): add periodic job to crawl pull requests and issues (#3449)" --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/" runUrl: "job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "c7a98766f1871e56194b77d54b648e08d499a31b" jenkins/label: "pingcap_tiflow_periodics_cdc_integration_mysql_test_234-jpjd1" name: "pingcap-tiflow-periodics-cdc-integration-mysql-test-234-j-vgmw9" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.23:latest" name: "golang" resources: limits: memory: "32Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-periodics-cdc-integration-mysql-test-234-j-vgmw9" - name: "JENKINS_NAME" value: "pingcap-tiflow-periodics-cdc-integration-mysql-test-234-j-vgmw9" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-periodics-cdc-integration-mysql-test-234-j-vgmw9 in /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test Agent pingcap-tiflow-periodics-cdc-integration-mysql-test-234-x-hbn5c is provisioned from template pingcap_tiflow_periodics_cdc_integration_mysql_test_234-xns4p-j15r7 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/" runUrl: "job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "ae54908fe2fb8aefccfed4ace52a0870dfe61d79" jenkins/label: "pingcap_tiflow_periodics_cdc_integration_mysql_test_234-xns4p" name: "pingcap-tiflow-periodics-cdc-integration-mysql-test-234-x-hbn5c" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.23:latest" name: "golang" resources: limits: memory: "32Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-periodics-cdc-integration-mysql-test-234-x-hbn5c" - name: "JENKINS_NAME" value: "pingcap-tiflow-periodics-cdc-integration-mysql-test-234-x-hbn5c" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-periodics-cdc-integration-mysql-test-234-x-hbn5c in /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test Agent pingcap-tiflow-periodics-cdc-integration-mysql-test-234-x-pbmg2 is provisioned from template pingcap_tiflow_periodics_cdc_integration_mysql_test_234-x37n0-x5mfl --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/" runUrl: "job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "8bfdec4db72eb095bf5d5749d8db3b9556d5d88e" jenkins/label: "pingcap_tiflow_periodics_cdc_integration_mysql_test_234-x37n0" name: "pingcap-tiflow-periodics-cdc-integration-mysql-test-234-x-pbmg2" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.23:latest" name: "golang" resources: limits: memory: "32Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-periodics-cdc-integration-mysql-test-234-x-pbmg2" - name: "JENKINS_NAME" value: "pingcap-tiflow-periodics-cdc-integration-mysql-test-234-x-pbmg2" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Agent pingcap-tiflow-periodics-cdc-integration-mysql-test-234-2-dwpgc is provisioned from template pingcap_tiflow_periodics_cdc_integration_mysql_test_234-20122-v61cg --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/" runUrl: "job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "d374db2646ad7fd4cc82369a2fd9b28884d31f33" jenkins/label: "pingcap_tiflow_periodics_cdc_integration_mysql_test_234-20122" name: "pingcap-tiflow-periodics-cdc-integration-mysql-test-234-2-dwpgc" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.23:latest" name: "golang" resources: limits: memory: "32Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-periodics-cdc-integration-mysql-test-234-2-dwpgc" - name: "JENKINS_NAME" value: "pingcap-tiflow-periodics-cdc-integration-mysql-test-234-2-dwpgc" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-periodics-cdc-integration-mysql-test-234-2-dwpgc in /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test Running on pingcap-tiflow-periodics-cdc-integration-mysql-test-234-x-pbmg2 in /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f b2a59508c11c86ba701c4af25fc536a9475b82c6 # timeout=10 Agent pingcap-tiflow-periodics-cdc-integration-mysql-test-234-j-x7t22 is provisioned from template pingcap_tiflow_periodics_cdc_integration_mysql_test_234-jn7pm-txqxp --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/" runUrl: "job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "7706b41f724f412ca1d7b2fc8f7d6708ce9f27fd" jenkins/label: "pingcap_tiflow_periodics_cdc_integration_mysql_test_234-jn7pm" name: "pingcap-tiflow-periodics-cdc-integration-mysql-test-234-j-x7t22" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.23:latest" name: "golang" resources: limits: memory: "32Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-periodics-cdc-integration-mysql-test-234-j-x7t22" - name: "JENKINS_NAME" value: "pingcap-tiflow-periodics-cdc-integration-mysql-test-234-j-x7t22" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-periodics-cdc-integration-mysql-test-234-j-x7t22 in /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f b2a59508c11c86ba701c4af25fc536a9475b82c6 # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f b2a59508c11c86ba701c4af25fc536a9475b82c6 # timeout=10 start tidb cluster in /tmp/tidb_cdc_test/down_db_require_secure_transport Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... Agent pingcap-tiflow-periodics-cdc-integration-mysql-test-234-z-4sl2f is provisioned from template pingcap_tiflow_periodics_cdc_integration_mysql_test_234-zrbwf-wrs0h --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/" runUrl: "job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "8398eb5396ba13e9804b96ba739efc34585f702a" jenkins/label: "pingcap_tiflow_periodics_cdc_integration_mysql_test_234-zrbwf" name: "pingcap-tiflow-periodics-cdc-integration-mysql-test-234-z-4sl2f" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.23:latest" name: "golang" resources: limits: memory: "32Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-periodics-cdc-integration-mysql-test-234-z-4sl2f" - name: "JENKINS_NAME" value: "pingcap-tiflow-periodics-cdc-integration-mysql-test-234-z-4sl2f" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-periodics-cdc-integration-mysql-test-234-z-4sl2f in /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test Agent pingcap-tiflow-periodics-cdc-integration-mysql-test-234-x-4tfw6 is provisioned from template pingcap_tiflow_periodics_cdc_integration_mysql_test_234-x0k9t-lmjv0 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/" runUrl: "job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "3433c2fab01c56c3a6fe6034a48b3a69ed99ba6a" jenkins/label: "pingcap_tiflow_periodics_cdc_integration_mysql_test_234-x0k9t" name: "pingcap-tiflow-periodics-cdc-integration-mysql-test-234-x-4tfw6" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.23:latest" name: "golang" resources: limits: memory: "32Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-periodics-cdc-integration-mysql-test-234-x-4tfw6" - name: "JENKINS_NAME" value: "pingcap-tiflow-periodics-cdc-integration-mysql-test-234-x-4tfw6" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-periodics-cdc-integration-mysql-test-234-x-4tfw6 in /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-periodics-cdc-integration-mysql-test-234-4-x87nv Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) config file /tmp/tidb_cdc_test/down_db_require_secure_transport/tidb-config-1743898647069779778.toml contained invalid configuration options: security.socket; check TiDB manual to make sure this option has not been deprecated and removed from your TiDB version if the option does not appear to be a typo Cache restored successfully (ws/jenkins-pingcap-tiflow-periodics_cdc_integration_mysql_test-234/tiflow-cdc) 4028702720 bytes in 8.15 secs (494594321 bytes/sec) [Pipeline] { [Pipeline] timeout Timeout set to expire in 45 min [Pipeline] { [Pipeline] sh [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] { + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G03 Run cases: row_format drop_many_tables processor_stop_delay partition_table ddl_with_exists PROW_JOB_ID=1ff7fcb7-127a-11f0-bebb-96e16cfa81f4 JENKINS_NODE_COOKIE=9dab791c-97f6-4b70-8b15-f6fa7740f668 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/ GOLANG_VERSION=1.23.3 HOSTNAME=pingcap-tiflow-periodics-cdc-integration-mysql-test-234-k-bx9s4 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-periodics_cdc_integration_mysql_test-234 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 JOB_SPEC={"type":"periodic","job":"pingcap/tiflow/periodics_cdc_integration_mysql_test","buildid":"1908671202193313792","prowjobid":"1ff7fcb7-127a-11f0-bebb-96e16cfa81f4","extra_refs":[{"org":"pingcap","repo":"tiflow","base_ref":"master"}]} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ JOB_NAME=pingcap/tiflow/periodics_cdc_integration_mysql_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#234 TEST_GROUP=G03 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1908671202193313792 GOLANG_DOWNLOAD_SHA256=a0afb9744c00648bafb1b90b4aba5bdb86f424f02f9275399ce0c20b93a2c3a8 JOB_BASE_NAME=periodics_cdc_integration_mysql_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_periodics_cdc_integration_mysql_test_234-krkrh GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-8f9954e717ae06be5a19c339deaa805ddd7d685252d85c8682580d893f46201d NODE_LABELS=pingcap-tiflow-periodics-cdc-integration-mysql-test-234-k-bx9s4 pingcap_tiflow_periodics_cdc_integration_mysql_test_234-krkrh GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-periodics-cdc-integration-mysql-test-234-k-bx9s4 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/display/redirect BUILD_NUMBER=234 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.23.3.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/row_format/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] checkout [Pipeline] checkout The recommended git tool is: git ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git The recommended git tool is: git [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@4dcb69a8; decorates RemoteLauncher[hudson.remoting.Channel@771a8545:JNLP4-connect connection from 10.233.78.130/10.233.78.130:59896] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 [Pipeline] stage [Pipeline] { (Test) [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow [Pipeline] { No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@2f985c75; decorates RemoteLauncher[hudson.remoting.Channel@1e42a1a2:JNLP4-connect connection from 10.233.77.50/10.233.77.50:59678] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow [Pipeline] { No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@69ad9a6a; decorates RemoteLauncher[hudson.remoting.Channel@139fc106:JNLP4-connect connection from 10.233.107.176/10.233.107.176:46532] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow [Pipeline] { No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@12462e59; decorates RemoteLauncher[hudson.remoting.Channel@598885e8:JNLP4-connect connection from 10.233.76.2/10.233.76.2:54450] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@4e9b2768; decorates RemoteLauncher[hudson.remoting.Channel@4d107f9b:JNLP4-connect connection from 10.233.76.109/10.233.76.109:49026] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow [Pipeline] { Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 [Pipeline] cache No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@3f1c1a3d; decorates RemoteLauncher[hudson.remoting.Channel@66fb0338:JNLP4-connect connection from 10.233.75.54/10.233.75.54:47266] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@239b0866; decorates RemoteLauncher[hudson.remoting.Channel@2149277a:JNLP4-connect connection from 10.233.123.185/10.233.123.185:49484] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@6547b0c7; decorates RemoteLauncher[hudson.remoting.Channel@3d6c64dd:JNLP4-connect connection from 10.233.126.34/10.233.126.34:36984] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@40b21d18; decorates RemoteLauncher[hudson.remoting.Channel@72e9385c:JNLP4-connect connection from 10.233.73.188/10.233.73.188:45802] will be ignored (a typical symptom is the Git executable not being run inside a designated container) No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@620c72f6; decorates RemoteLauncher[hudson.remoting.Channel@1dc89f12:JNLP4-connect connection from 10.233.75.191/10.233.75.191:45398] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@18d4680a; decorates RemoteLauncher[hudson.remoting.Channel@11a9f2d5:JNLP4-connect connection from 10.233.123.243/10.233.123.243:34750] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@36f7975e; decorates RemoteLauncher[hudson.remoting.Channel@680f0e08:JNLP4-connect connection from 10.233.72.245/10.233.72.245:60868] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489987642628773417 TiDB Cluster ID. tikv_gc_leader_uuid 65821d68ac00006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-9-92vm1, pid:1352, start at 2025-04-06 08:17:29.271437446 +0800 CST m=+2.251369976 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:19:29.279 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:17:29.264 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:07:29.264 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489987642628773417 TiDB Cluster ID. tikv_gc_leader_uuid 65821d68ac00006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-9-92vm1, pid:1352, start at 2025-04-06 08:17:29.271437446 +0800 CST m=+2.251369976 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:19:29.279 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:17:29.264 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:07:29.264 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489987643413064618 TiDB Cluster ID. tikv_gc_leader_uuid 65821d68c480002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-9-92vm1, pid:1428, start at 2025-04-06 08:17:29.363875567 +0800 CST m=+2.286831245 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:19:29.371 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:17:29.362 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:07:29.362 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/down_db_require_secure_transport/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/down_db_require_secure_transport/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/down_db_require_secure_transport/tiflash/db/proxy"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/down_db_require_secure_transport/tiflash/log/proxy.log"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/down_db_require_secure_transport/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } > git init /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Still waiting to schedule task ‘pingcap-tiflow-periodics-cdc-integration-mysql-test-234-4-x87nv’ is offline Avoid second fetch Checking out Revision b2a59508c11c86ba701c4af25fc536a9475b82c6 (origin/main) Agent pingcap-tiflow-periodics-cdc-integration-mysql-test-234-4-x87nv is provisioned from template pingcap_tiflow_periodics_cdc_integration_mysql_test_234-44vnf-m89qw --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/" runUrl: "job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "c8c2939b5077330ad21b129fd1ed7f50ecaa5720" jenkins/label: "pingcap_tiflow_periodics_cdc_integration_mysql_test_234-44vnf" name: "pingcap-tiflow-periodics-cdc-integration-mysql-test-234-4-x87nv" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.23:latest" name: "golang" resources: limits: memory: "32Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-periodics-cdc-integration-mysql-test-234-4-x87nv" - name: "JENKINS_NAME" value: "pingcap-tiflow-periodics-cdc-integration-mysql-test-234-4-x87nv" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-periodics-cdc-integration-mysql-test-234-4-x87nv in /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test Commit message: "feat(prow-jobs/pingcap-qe/ci): add periodic job to crawl pull requests and issues (#3449)" Avoid second fetch Checking out Revision b2a59508c11c86ba701c4af25fc536a9475b82c6 (origin/main) Avoid second fetch Checking out Revision b2a59508c11c86ba701c4af25fc536a9475b82c6 (origin/main) > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f b2a59508c11c86ba701c4af25fc536a9475b82c6 # timeout=10 Avoid second fetch Checking out Revision b2a59508c11c86ba701c4af25fc536a9475b82c6 (origin/main) > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f b2a59508c11c86ba701c4af25fc536a9475b82c6 # timeout=10 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 Avoid second fetch Checking out Revision b2a59508c11c86ba701c4af25fc536a9475b82c6 (origin/main) Avoid second fetch Checking out Revision b2a59508c11c86ba701c4af25fc536a9475b82c6 (origin/main) Commit message: "feat(prow-jobs/pingcap-qe/ci): add periodic job to crawl pull requests and issues (#3449)" Avoid second fetch Checking out Revision b2a59508c11c86ba701c4af25fc536a9475b82c6 (origin/main) > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f b2a59508c11c86ba701c4af25fc536a9475b82c6 # timeout=10 Avoid second fetch Checking out Revision b2a59508c11c86ba701c4af25fc536a9475b82c6 (origin/main) Commit message: "feat(prow-jobs/pingcap-qe/ci): add periodic job to crawl pull requests and issues (#3449)" > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f b2a59508c11c86ba701c4af25fc536a9475b82c6 # timeout=10 Avoid second fetch Checking out Revision b2a59508c11c86ba701c4af25fc536a9475b82c6 (origin/main) Commit message: "feat(prow-jobs/pingcap-qe/ci): add periodic job to crawl pull requests and issues (#3449)" > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f b2a59508c11c86ba701c4af25fc536a9475b82c6 # timeout=10 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f b2a59508c11c86ba701c4af25fc536a9475b82c6 # timeout=10 Commit message: "feat(prow-jobs/pingcap-qe/ci): add periodic job to crawl pull requests and issues (#3449)" Commit message: "feat(prow-jobs/pingcap-qe/ci): add periodic job to crawl pull requests and issues (#3449)" Avoid second fetch Checking out Revision b2a59508c11c86ba701c4af25fc536a9475b82c6 (origin/main) Commit message: "feat(prow-jobs/pingcap-qe/ci): add periodic job to crawl pull requests and issues (#3449)" Commit message: "feat(prow-jobs/pingcap-qe/ci): add periodic job to crawl pull requests and issues (#3449)" start tidb cluster in /tmp/tidb_cdc_test/row_format Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f b2a59508c11c86ba701c4af25fc536a9475b82c6 # timeout=10 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 Avoid second fetch > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f b2a59508c11c86ba701c4af25fc536a9475b82c6 # timeout=10 Checking out Revision b2a59508c11c86ba701c4af25fc536a9475b82c6 (origin/main) > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f b2a59508c11c86ba701c4af25fc536a9475b82c6 # timeout=10 Commit message: "feat(prow-jobs/pingcap-qe/ci): add periodic job to crawl pull requests and issues (#3449)" Commit message: "feat(prow-jobs/pingcap-qe/ci): add periodic job to crawl pull requests and issues (#3449)" Commit message: "feat(prow-jobs/pingcap-qe/ci): add periodic job to crawl pull requests and issues (#3449)" [Sun Apr 6 08:17:33 CST 2025] <<<<<< START cdc server in down_db_require_secure_transport case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.down_db_require_secure_transport.26712673.out server --log-file /tmp/tidb_cdc_test/down_db_require_secure_transport/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/down_db_require_secure_transport/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f b2a59508c11c86ba701c4af25fc536a9475b82c6 # timeout=10 Avoid second fetch Checking out Revision b2a59508c11c86ba701c4af25fc536a9475b82c6 (origin/main) Verifying downstream PD is started... Commit message: "feat(prow-jobs/pingcap-qe/ci): add periodic job to crawl pull requests and issues (#3449)" > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f b2a59508c11c86ba701c4af25fc536a9475b82c6 # timeout=10 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f b2a59508c11c86ba701c4af25fc536a9475b82c6 # timeout=10 Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 06 Apr 2025 00:17:36 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/26f3734f-a7d8-441d-9fa0-30db6b16074e {"id":"26f3734f-a7d8-441d-9fa0-30db6b16074e","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898653} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/2231960875820dd6 26f3734f-a7d8-441d-9fa0-30db6b16074e /tidb/cdc/default/default/upstream/7489987642628773417 {"id":7489987642628773417,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/26f3734f-a7d8-441d-9fa0-30db6b16074e {"id":"26f3734f-a7d8-441d-9fa0-30db6b16074e","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898653} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/2231960875820dd6 26f3734f-a7d8-441d-9fa0-30db6b16074e /tidb/cdc/default/default/upstream/7489987642628773417 {"id":7489987642628773417,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/26f3734f-a7d8-441d-9fa0-30db6b16074e {"id":"26f3734f-a7d8-441d-9fa0-30db6b16074e","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898653} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/2231960875820dd6 26f3734f-a7d8-441d-9fa0-30db6b16074e /tidb/cdc/default/default/upstream/7489987642628773417 {"id":7489987642628773417,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.down_db_require_secure_transport.cli.2742.out cli changefeed create '--sink-uri=mysql://root@127.0.0.1:3306/?ssl-ca=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem&ssl-key=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client-key.pem&ssl-cert=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client.pem' --changefeed-id=down-db-require-secure-transport Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Create changefeed successfully! ID: down-db-require-secure-transport Info: {"upstream_id":7489987642628773417,"namespace":"default","id":"down-db-require-secure-transport","sink_uri":"mysql://root@127.0.0.1:3306/?ssl-ca=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem\u0026ssl-key=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client-key.pem\u0026ssl-cert=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client.pem","create_time":"2025-04-06T08:17:38.979640055+08:00","start_ts":457152570016399365,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457152570016399365,"checkpoint_ts":457152570016399365,"checkpoint_time":"2025-04-06 08:17:38.815"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... Cache restored successfully (ws/jenkins-pingcap-tiflow-periodics_cdc_integration_mysql_test-234/tiflow-cdc) 4028702720 bytes in 7.70 secs (523516660 bytes/sec) [Pipeline] { [Pipeline] cache + set +x ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table down_db_require_secure_transport.finishmark not exists for 1-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table down_db_require_secure_transport.finishmark exists [Sun Apr 6 08:17:44 CST 2025] <<<<<< run test case down_db_require_secure_transport success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Cache restored successfully (ws/jenkins-pingcap-tiflow-periodics_cdc_integration_mysql_test-234/tiflow-cdc) 4028702720 bytes in 7.41 secs (543655616 bytes/sec) [Pipeline] { [Pipeline] cache ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/bdr_mode/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... Cache restored successfully (ws/jenkins-pingcap-tiflow-periodics_cdc_integration_mysql_test-234/tiflow-cdc) 4028702720 bytes in 7.66 secs (525944461 bytes/sec) [Pipeline] { [Pipeline] cache ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/bdr_mode Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489987680327643760 TiDB Cluster ID. tikv_gc_leader_uuid 65821d89630000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-k-bx9s4, pid:1278, start at 2025-04-06 08:18:02.814351373 +0800 CST m=+24.306000228 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:20:02.822 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:18:02.814 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:08:02.814 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489987680327643760 TiDB Cluster ID. tikv_gc_leader_uuid 65821d89630000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-k-bx9s4, pid:1278, start at 2025-04-06 08:18:02.814351373 +0800 CST m=+24.306000228 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:20:02.822 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:18:02.814 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:08:02.814 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489987693547576704 TiDB Cluster ID. tikv_gc_leader_uuid 65821d73b54000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-k-bx9s4, pid:1344, start at 2025-04-06 08:17:40.614062462 +0800 CST m=+2.031188349 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:19:40.623 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:17:40.615 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:07:40.615 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/row_format/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/row_format/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/row_format/tiflash/db/proxy"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/row_format/tiflash/log/proxy.log"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/row_format/tiflash-proxy.toml"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.row_format.cli.2725.out cli tso query --pd=http://127.0.0.1:2379 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x + tso='457152577474396161 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 457152577474396161 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Sun Apr 6 08:18:08 CST 2025] <<<<<< START cdc server in row_format case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.row_format.27582760.out server --log-file /tmp/tidb_cdc_test/row_format/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/row_format/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489987811229278668 TiDB Cluster ID. tikv_gc_leader_uuid 65821d8f72c0008 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-9-92vm1, pid:3724, start at 2025-04-06 08:18:08.98224353 +0800 CST m=+2.256188767 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:20:09.035 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:18:09.030 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:08:09.030 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489987811229278668 TiDB Cluster ID. tikv_gc_leader_uuid 65821d8f72c0008 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-9-92vm1, pid:3724, start at 2025-04-06 08:18:08.98224353 +0800 CST m=+2.256188767 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:20:09.035 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:18:09.030 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:08:09.030 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489987814565995235 TiDB Cluster ID. tikv_gc_leader_uuid 65821d8f688000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-9-92vm1, pid:3800, start at 2025-04-06 08:18:08.965530991 +0800 CST m=+2.185609160 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:20:08.973 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:18:08.967 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:08:08.967 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/bdr_mode/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/bdr_mode/tiflash/log/error.log arg matches is ArgMatches { args: {"pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/bdr_mode/tiflash-proxy.toml"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/bdr_mode/tiflash/db/proxy"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/bdr_mode/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Cache restored successfully (ws/jenkins-pingcap-tiflow-periodics_cdc_integration_mysql_test-234/tiflow-cdc) 4028702720 bytes in 10.64 secs (378523461 bytes/sec) [Pipeline] { [Pipeline] cache + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 06 Apr 2025 00:18:11 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e399e7d5-1199-4030-8167-252b338308ae {"id":"e399e7d5-1199-4030-8167-252b338308ae","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898689} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/2231960875aec842 e399e7d5-1199-4030-8167-252b338308ae /tidb/cdc/default/default/upstream/7489987680327643760 {"id":7489987680327643760,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e399e7d5-1199-4030-8167-252b338308ae {"id":"e399e7d5-1199-4030-8167-252b338308ae","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898689} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/2231960875aec842 e399e7d5-1199-4030-8167-252b338308ae /tidb/cdc/default/default/upstream/7489987680327643760 {"id":7489987680327643760,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e399e7d5-1199-4030-8167-252b338308ae {"id":"e399e7d5-1199-4030-8167-252b338308ae","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898689} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/2231960875aec842 e399e7d5-1199-4030-8167-252b338308ae /tidb/cdc/default/default/upstream/7489987680327643760 {"id":7489987680327643760,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.row_format.cli.2812.out cli changefeed create --start-ts=457152577474396161 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ Create changefeed successfully! ID: f504c192-2812-4d6f-95ba-9ed995c38c3c Info: {"upstream_id":7489987680327643760,"namespace":"default","id":"f504c192-2812-4d6f-95ba-9ed995c38c3c","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-06T08:18:12.258002391+08:00","start_ts":457152577474396161,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457152577474396161,"checkpoint_ts":457152577474396161,"checkpoint_time":"2025-04-06 08:18:07.265"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... [Sun Apr 6 08:18:13 CST 2025] <<<<<< START cdc server in bdr_mode case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.bdr_mode.52075209.out server --log-file /tmp/tidb_cdc_test/bdr_mode/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/bdr_mode/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + set +x + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 06 Apr 2025 00:18:16 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3853c5ae-adb3-43a0-972a-a7abb0b90afd {"id":"3853c5ae-adb3-43a0-972a-a7abb0b90afd","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898693} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319608762110d7 3853c5ae-adb3-43a0-972a-a7abb0b90afd /tidb/cdc/default/default/upstream/7489987811229278668 {"id":7489987811229278668,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3853c5ae-adb3-43a0-972a-a7abb0b90afd {"id":"3853c5ae-adb3-43a0-972a-a7abb0b90afd","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898693} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319608762110d7 3853c5ae-adb3-43a0-972a-a7abb0b90afd /tidb/cdc/default/default/upstream/7489987811229278668 {"id":7489987811229278668,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3853c5ae-adb3-43a0-972a-a7abb0b90afd {"id":"3853c5ae-adb3-43a0-972a-a7abb0b90afd","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898693} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319608762110d7 3853c5ae-adb3-43a0-972a-a7abb0b90afd /tidb/cdc/default/default/upstream/7489987811229278668 {"id":7489987811229278668,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Sun Apr 6 08:18:16 CST 2025] <<<<<< START cdc server in bdr_mode case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8400/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.bdr_mode.52695271.out server --log-file /tmp/tidb_cdc_test/bdr_mode/cdcdown.log --log-level debug --data-dir /tmp/tidb_cdc_test/bdr_mode/cdc_datadown --cluster-id default --addr 127.0.0.1:8400 --pd http://127.0.0.1:2479 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8400/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8400 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8400; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8400/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8400 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8400 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8400 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 06 Apr 2025 00:18:19 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6adb9e25-1fe2-415d-9e24-24f4afff3446 {"id":"6adb9e25-1fe2-415d-9e24-24f4afff3446","address":"127.0.0.1:8400","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898696} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/3511960876213f01 6adb9e25-1fe2-415d-9e24-24f4afff3446 /tidb/cdc/default/default/upstream/7489987814565995235 {"id":7489987814565995235,"pd-endpoints":"http://127.0.0.1:2479,http://127.0.0.1:2479","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6adb9e25-1fe2-415d-9e24-24f4afff3446 {"id":"6adb9e25-1fe2-415d-9e24-24f4afff3446","address":"127.0.0.1:8400","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898696} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/3511960876213f01 6adb9e25-1fe2-415d-9e24-24f4afff3446 /tidb/cdc/default/default/upstream/7489987814565995235 {"id":7489987814565995235,"pd-endpoints":"http://127.0.0.1:2479,http://127.0.0.1:2479","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6adb9e25-1fe2-415d-9e24-24f4afff3446 {"id":"6adb9e25-1fe2-415d-9e24-24f4afff3446","address":"127.0.0.1:8400","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898696} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/3511960876213f01 6adb9e25-1fe2-415d-9e24-24f4afff3446 /tidb/cdc/default/default/upstream/7489987814565995235 {"id":7489987814565995235,"pd-endpoints":"http://127.0.0.1:2479,http://127.0.0.1:2479","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.bdr_mode.cli.5328.out cli changefeed create --sink-uri=mysql://root@127.0.0.1:3306 -c test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/bdr_mode/conf/up.toml Create changefeed successfully! ID: test-1 Info: {"upstream_id":7489987811229278668,"namespace":"default","id":"test-1","sink_uri":"mysql://root@127.0.0.1:3306","create_time":"2025-04-06T08:18:19.666781621+08:00","start_ts":457152580700602373,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":true,"enable_table_monitor":false,"bdr_mode":true,"sync_point_interval":30000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457152580700602373,"checkpoint_ts":457152580700602373,"checkpoint_time":"2025-04-06 08:18:19.572"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.bdr_mode.cli.5375.out cli changefeed create --sink-uri=mysql://root@127.0.0.1:4000 -c test-2 --server http://127.0.0.1:8400 --config=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/bdr_mode/conf/down.toml Create changefeed successfully! ID: test-2 Info: {"upstream_id":7489987814565995235,"namespace":"default","id":"test-2","sink_uri":"mysql://root@127.0.0.1:4000","create_time":"2025-04-06T08:18:21.547266742+08:00","start_ts":457152581184520196,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":true,"enable_table_monitor":false,"bdr_mode":true,"sync_point_interval":30000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457152581184520196,"checkpoint_ts":457152581184520196,"checkpoint_time":"2025-04-06 08:18:21.418"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x table row_format.finish_mark not exists for 1-th check, retry later Cache restored successfully (ws/jenkins-pingcap-tiflow-periodics_cdc_integration_mysql_test-234/tiflow-cdc) 4028702720 bytes in 10.45 secs (385671035 bytes/sec) [Pipeline] { [Pipeline] cache table row_format.finish_mark not exists for 2-th check, retry later table row_format.finish_mark not exists for 3-th check, retry later table row_format.finish_mark not exists for 4-th check, retry later table row_format.finish_mark not exists for 5-th check, retry later table row_format.finish_mark not exists for 6-th check, retry later Cache restored successfully (ws/jenkins-pingcap-tiflow-periodics_cdc_integration_mysql_test-234/tiflow-cdc) 4028702720 bytes in 6.77 secs (594887319 bytes/sec) [Pipeline] { [Pipeline] cache table row_format.finish_mark not exists for 7-th check, retry later table row_format.finish_mark not exists for 8-th check, retry later table row_format.finish_mark not exists for 9-th check, retry later table row_format.finish_mark not exists for 10-th check, retry later table row_format.finish_mark exists check diff successfully wait process cdc.test exit for 1-th time... Cache restored successfully (ws/jenkins-pingcap-tiflow-periodics_cdc_integration_mysql_test-234/tiflow-cdc) 4028702720 bytes in 7.97 secs (505315826 bytes/sec) [Pipeline] { [Pipeline] timeout Timeout set to expire in 45 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 45 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 45 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 45 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 45 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 45 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 45 min [Pipeline] { [Pipeline] sh wait process cdc.test exit for 2-th time... [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G06 Run cases: sink_retry changefeed_error ddl_sequence resourcecontrol [Pipeline] sh cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Sun Apr 6 08:18:44 CST 2025] <<<<<< run test case row_format success! >>>>>> PROW_JOB_ID=1ff7fcb7-127a-11f0-bebb-96e16cfa81f4 JENKINS_NODE_COOKIE=87eb8ba8-c722-48ee-b1c0-47310bd5fc33 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/ GOLANG_VERSION=1.23.3 HOSTNAME=pingcap-tiflow-periodics-cdc-integration-mysql-test-234-m-cm2pq HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-periodics_cdc_integration_mysql_test-234 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 JOB_SPEC={"type":"periodic","job":"pingcap/tiflow/periodics_cdc_integration_mysql_test","buildid":"1908671202193313792","prowjobid":"1ff7fcb7-127a-11f0-bebb-96e16cfa81f4","extra_refs":[{"org":"pingcap","repo":"tiflow","base_ref":"master"}]} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ JOB_NAME=pingcap/tiflow/periodics_cdc_integration_mysql_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#234 TEST_GROUP=G06 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1908671202193313792 GOLANG_DOWNLOAD_SHA256=a0afb9744c00648bafb1b90b4aba5bdb86f424f02f9275399ce0c20b93a2c3a8 JOB_BASE_NAME=periodics_cdc_integration_mysql_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_periodics_cdc_integration_mysql_test_234-m9519 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-8f9954e717ae06be5a19c339deaa805ddd7d685252d85c8682580d893f46201d NODE_LABELS=pingcap-tiflow-periodics-cdc-integration-mysql-test-234-m-cm2pq pingcap_tiflow_periodics_cdc_integration_mysql_test_234-m9519 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-periodics-cdc-integration-mysql-test-234-m-cm2pq GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/display/redirect BUILD_NUMBER=234 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.23.3.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/sink_retry/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G04 Run cases: foreign_key ddl_puller_lag ddl_only_block_related_table changefeed_auto_stop PROW_JOB_ID=1ff7fcb7-127a-11f0-bebb-96e16cfa81f4 JENKINS_NODE_COOKIE=93cdf146-492a-4011-845f-f0bc27cad914 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/ GOLANG_VERSION=1.23.3 HOSTNAME=pingcap-tiflow-periodics-cdc-integration-mysql-test-234-c-m09xf HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-periodics_cdc_integration_mysql_test-234 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 JOB_SPEC={"type":"periodic","job":"pingcap/tiflow/periodics_cdc_integration_mysql_test","buildid":"1908671202193313792","prowjobid":"1ff7fcb7-127a-11f0-bebb-96e16cfa81f4","extra_refs":[{"org":"pingcap","repo":"tiflow","base_ref":"master"}]} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ JOB_NAME=pingcap/tiflow/periodics_cdc_integration_mysql_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#234 TEST_GROUP=G04 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1908671202193313792 GOLANG_DOWNLOAD_SHA256=a0afb9744c00648bafb1b90b4aba5bdb86f424f02f9275399ce0c20b93a2c3a8 JOB_BASE_NAME=periodics_cdc_integration_mysql_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_periodics_cdc_integration_mysql_test_234-cf22g GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-8f9954e717ae06be5a19c339deaa805ddd7d685252d85c8682580d893f46201d NODE_LABELS=pingcap-tiflow-periodics-cdc-integration-mysql-test-234-c-m09xf pingcap_tiflow_periodics_cdc_integration_mysql_test_234-cf22g GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-periodics-cdc-integration-mysql-test-234-c-m09xf GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/display/redirect BUILD_NUMBER=234 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.23.3.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/foreign_key/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G01 Run cases: http_api http_api_tls api_v2 http_api_tls_with_user_auth cli_tls_with_auth debezium_basic kafka_simple_basic kafka_simple_basic_avro kafka_simple_handle_key_only kafka_simple_handle_key_only_avro kafka_simple_claim_check kafka_simple_claim_check_avro canal_json_adapter_compatibility canal_json_basic canal_json_content_compatible multi_topics avro_basic canal_json_handle_key_only open_protocol_handle_key_only canal_json_claim_check open_protocol_claim_check canal_json_storage_basic canal_json_storage_partition_table multi_tables_ddl PROW_JOB_ID=1ff7fcb7-127a-11f0-bebb-96e16cfa81f4 JENKINS_NODE_COOKIE=14fc89b2-89bc-43b1-a0a7-f0514114f511 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/ GOLANG_VERSION=1.23.3 HOSTNAME=pingcap-tiflow-periodics-cdc-integration-mysql-test-234-8-31x9h HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-periodics_cdc_integration_mysql_test-234 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 JOB_SPEC={"type":"periodic","job":"pingcap/tiflow/periodics_cdc_integration_mysql_test","buildid":"1908671202193313792","prowjobid":"1ff7fcb7-127a-11f0-bebb-96e16cfa81f4","extra_refs":[{"org":"pingcap","repo":"tiflow","base_ref":"master"}]} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ JOB_NAME=pingcap/tiflow/periodics_cdc_integration_mysql_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#234 TEST_GROUP=G01 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1908671202193313792 GOLANG_DOWNLOAD_SHA256=a0afb9744c00648bafb1b90b4aba5bdb86f424f02f9275399ce0c20b93a2c3a8 JOB_BASE_NAME=periodics_cdc_integration_mysql_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_periodics_cdc_integration_mysql_test_234-84pgq GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-8f9954e717ae06be5a19c339deaa805ddd7d685252d85c8682580d893f46201d NODE_LABELS=pingcap_tiflow_periodics_cdc_integration_mysql_test_234-84pgq pingcap-tiflow-periodics-cdc-integration-mysql-test-234-8-31x9h GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-periodics-cdc-integration-mysql-test-234-8-31x9h GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/display/redirect BUILD_NUMBER=234 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.23.3.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/http_api/run.sh using Sink-Type: mysql... <<================= [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G05 [Pipeline] sh Run cases: charset_gbk ddl_manager multi_source vector PROW_JOB_ID=1ff7fcb7-127a-11f0-bebb-96e16cfa81f4 JENKINS_NODE_COOKIE=03c0da79-8435-4480-b3fe-414f37054a76 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/ GOLANG_VERSION=1.23.3 HOSTNAME=pingcap-tiflow-periodics-cdc-integration-mysql-test-234-v-hm9ds HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-periodics_cdc_integration_mysql_test-234 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 JOB_SPEC={"type":"periodic","job":"pingcap/tiflow/periodics_cdc_integration_mysql_test","buildid":"1908671202193313792","prowjobid":"1ff7fcb7-127a-11f0-bebb-96e16cfa81f4","extra_refs":[{"org":"pingcap","repo":"tiflow","base_ref":"master"}]} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ JOB_NAME=pingcap/tiflow/periodics_cdc_integration_mysql_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#234 TEST_GROUP=G05 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1908671202193313792 GOLANG_DOWNLOAD_SHA256=a0afb9744c00648bafb1b90b4aba5bdb86f424f02f9275399ce0c20b93a2c3a8 JOB_BASE_NAME=periodics_cdc_integration_mysql_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_periodics_cdc_integration_mysql_test_234-vf84t GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-8f9954e717ae06be5a19c339deaa805ddd7d685252d85c8682580d893f46201d NODE_LABELS=pingcap_tiflow_periodics_cdc_integration_mysql_test_234-vf84t pingcap-tiflow-periodics-cdc-integration-mysql-test-234-v-hm9ds GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-periodics-cdc-integration-mysql-test-234-v-hm9ds GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/display/redirect BUILD_NUMBER=234 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.23.3.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/charset_gbk/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... WARNING: Running pip install with root privileges is generally not a good idea. Try `__main__.py install --user` instead. Collecting requests==2.26.0 + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G07 Run cases: kv_client_stream_reconnect cdc split_region PROW_JOB_ID=1ff7fcb7-127a-11f0-bebb-96e16cfa81f4 JENKINS_NODE_COOKIE=94b47f50-3771-44ad-b092-2517e1c25556 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/ GOLANG_VERSION=1.23.3 HOSTNAME=pingcap-tiflow-periodics-cdc-integration-mysql-test-234-v-nsng0 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-periodics_cdc_integration_mysql_test-234 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 JOB_SPEC={"type":"periodic","job":"pingcap/tiflow/periodics_cdc_integration_mysql_test","buildid":"1908671202193313792","prowjobid":"1ff7fcb7-127a-11f0-bebb-96e16cfa81f4","extra_refs":[{"org":"pingcap","repo":"tiflow","base_ref":"master"}]} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ JOB_NAME=pingcap/tiflow/periodics_cdc_integration_mysql_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#234 TEST_GROUP=G07 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1908671202193313792 GOLANG_DOWNLOAD_SHA256=a0afb9744c00648bafb1b90b4aba5bdb86f424f02f9275399ce0c20b93a2c3a8 JOB_BASE_NAME=periodics_cdc_integration_mysql_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_periodics_cdc_integration_mysql_test_234-vvls1 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-8f9954e717ae06be5a19c339deaa805ddd7d685252d85c8682580d893f46201d NODE_LABELS=pingcap_tiflow_periodics_cdc_integration_mysql_test_234-vvls1 pingcap-tiflow-periodics-cdc-integration-mysql-test-234-v-nsng0 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-periodics-cdc-integration-mysql-test-234-v-nsng0 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/display/redirect BUILD_NUMBER=234 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.23.3.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/kv_client_stream_reconnect/run.sh using Sink-Type: mysql... <<================= [Pipeline] sh The 1 times to try to start tidb cluster... + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G02 Run cases: consistent_replicate_ddl consistent_replicate_gbk consistent_replicate_nfs consistent_replicate_storage_file consistent_replicate_storage_file_large_value consistent_replicate_storage_s3 consistent_partition_table kafka_big_messages_v2 multi_tables_ddl_v2 multi_topics_v2 storage_cleanup csv_storage_basic csv_storage_multi_tables_ddl csv_storage_partition_table PROW_JOB_ID=1ff7fcb7-127a-11f0-bebb-96e16cfa81f4 JENKINS_NODE_COOKIE=49f095a0-67d2-459a-a648-97cca9305da4 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/ GOLANG_VERSION=1.23.3 HOSTNAME=pingcap-tiflow-periodics-cdc-integration-mysql-test-234-6-zhp59 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-periodics_cdc_integration_mysql_test-234 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 JOB_SPEC={"type":"periodic","job":"pingcap/tiflow/periodics_cdc_integration_mysql_test","buildid":"1908671202193313792","prowjobid":"1ff7fcb7-127a-11f0-bebb-96e16cfa81f4","extra_refs":[{"org":"pingcap","repo":"tiflow","base_ref":"master"}]} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ JOB_NAME=pingcap/tiflow/periodics_cdc_integration_mysql_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#234 TEST_GROUP=G02 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1908671202193313792 GOLANG_DOWNLOAD_SHA256=a0afb9744c00648bafb1b90b4aba5bdb86f424f02f9275399ce0c20b93a2c3a8 JOB_BASE_NAME=periodics_cdc_integration_mysql_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_periodics_cdc_integration_mysql_test_234-693tt GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-8f9954e717ae06be5a19c339deaa805ddd7d685252d85c8682580d893f46201d NODE_LABELS=pingcap-tiflow-periodics-cdc-integration-mysql-test-234-6-zhp59 pingcap_tiflow_periodics_cdc_integration_mysql_test_234-693tt GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-periodics-cdc-integration-mysql-test-234-6-zhp59 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/display/redirect BUILD_NUMBER=234 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.23.3.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/consistent_replicate_ddl/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G08 Run cases: processor_err_chan changefeed_reconstruct multi_capture synced_status_with_redo PROW_JOB_ID=1ff7fcb7-127a-11f0-bebb-96e16cfa81f4 JENKINS_NODE_COOKIE=426df6c4-f3f0-4d45-8d38-150f8e2d7df6 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/ GOLANG_VERSION=1.23.3 HOSTNAME=pingcap-tiflow-periodics-cdc-integration-mysql-test-234-d-7dnbd HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-periodics_cdc_integration_mysql_test-234 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 JOB_SPEC={"type":"periodic","job":"pingcap/tiflow/periodics_cdc_integration_mysql_test","buildid":"1908671202193313792","prowjobid":"1ff7fcb7-127a-11f0-bebb-96e16cfa81f4","extra_refs":[{"org":"pingcap","repo":"tiflow","base_ref":"master"}]} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ JOB_NAME=pingcap/tiflow/periodics_cdc_integration_mysql_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#234 TEST_GROUP=G08 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1908671202193313792 GOLANG_DOWNLOAD_SHA256=a0afb9744c00648bafb1b90b4aba5bdb86f424f02f9275399ce0c20b93a2c3a8 JOB_BASE_NAME=periodics_cdc_integration_mysql_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_periodics_cdc_integration_mysql_test_234-dbhbr GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-8f9954e717ae06be5a19c339deaa805ddd7d685252d85c8682580d893f46201d NODE_LABELS=pingcap-tiflow-periodics-cdc-integration-mysql-test-234-d-7dnbd pingcap_tiflow_periodics_cdc_integration_mysql_test_234-dbhbr GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-periodics-cdc-integration-mysql-test-234-d-7dnbd GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/display/redirect BUILD_NUMBER=234 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.23.3.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/processor_err_chan/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... Downloading https://files.pythonhosted.org/packages/92/96/144f70b972a9c0eabbd4391ef93ccd49d0f2747f4f6a2a2738e99e5adc65/requests-2.26.0-py2.py3-none-any.whl (62kB) Collecting idna<4,>=2.5; python_version >= "3" (from requests==2.26.0) Downloading https://files.pythonhosted.org/packages/76/c6/c88e154df9c4e1a2a66ccf0005a88dfb2650c1dffb6f5ce603dfbd452ce3/idna-3.10-py3-none-any.whl (70kB) Collecting urllib3<1.27,>=1.21.1 (from requests==2.26.0) Downloading https://files.pythonhosted.org/packages/33/cf/8435d5a7159e2a9c83a95896ed596f68cf798005fe107cc655b5c5c14704/urllib3-1.26.20-py2.py3-none-any.whl (144kB) [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { Collecting certifi>=2017.4.17 (from requests==2.26.0) Downloading https://files.pythonhosted.org/packages/38/fc/bce832fd4fd99766c04d1ee0eead6b0ec6486fb100ae5e74c1d91292b982/certifi-2025.1.31-py3-none-any.whl (166kB) [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { Requirement already up-to-date: charset-normalizer~=2.0.0; python_version >= "3" in /usr/local/lib/python3.6/site-packages (from requests==2.26.0) Installing collected packages: idna, urllib3, certifi, requests Found existing installation: idna 3.4 Uninstalling idna-3.4: Successfully uninstalled idna-3.4 [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { Found existing installation: urllib3 1.26.16 Uninstalling urllib3-1.26.16: Successfully uninstalled urllib3-1.26.16 Found existing installation: certifi 2023.5.7 Uninstalling certifi-2023.5.7: Successfully uninstalled certifi-2023.5.7 Found existing installation: requests 2.27.1 Uninstalling requests-2.27.1: Successfully uninstalled requests-2.27.1 Successfully installed certifi-2025.1.31 idna-3.10 requests-2.26.0 urllib3-1.26.20 start tidb cluster in /tmp/tidb_cdc_test/foreign_key Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/sink_retry Starting Upstream PD... [Pipeline] { [Pipeline] checkout The recommended git tool is: git [Pipeline] stage [Pipeline] { (Test) start tidb cluster in /tmp/tidb_cdc_test/kv_client_stream_reconnect Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) The 1 times to try to start tidb cluster... [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) start tidb cluster in /tmp/tidb_cdc_test/processor_err_chan Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@6c542cf9; decorates RemoteLauncher[hudson.remoting.Channel@49f6dbf0:JNLP4-connect connection from 10.233.77.234/10.233.77.234:34258] will be ignored (a typical symptom is the Git executable not being run inside a designated container) [Pipeline] { Cloning the remote Git repository Using shallow clone with depth 1 [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow [Pipeline] { start tidb cluster in /tmp/tidb_cdc_test/consistent_replicate_ddl Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... [Pipeline] cache Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Verifying downstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release start tidb cluster in /tmp/tidb_cdc_test/charset_gbk Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... Verifying downstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Avoid second fetch Checking out Revision b2a59508c11c86ba701c4af25fc536a9475b82c6 (origin/main) Commit message: "feat(prow-jobs/pingcap-qe/ci): add periodic job to crawl pull requests and issues (#3449)" > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f b2a59508c11c86ba701c4af25fc536a9475b82c6 # timeout=10 Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release start tidb cluster in /tmp/tidb_cdc_test/http_api Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/drop_many_tables/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489988010948606366 TiDB Cluster ID. tikv_gc_leader_uuid 65821dbbaa0000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-v-nsng0, pid:1358, start at 2025-04-06 08:18:54.291508421 +0800 CST m=+2.128187630 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:20:54.334 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:18:54.299 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:08:54.299 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489988010948606366 TiDB Cluster ID. tikv_gc_leader_uuid 65821dbbaa0000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-v-nsng0, pid:1358, start at 2025-04-06 08:18:54.291508421 +0800 CST m=+2.128187630 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:20:54.334 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:18:54.299 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:08:54.299 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489988011889615551 TiDB Cluster ID. tikv_gc_leader_uuid 65821dbbdb0000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-v-nsng0, pid:1425, start at 2025-04-06 08:18:54.479934971 +0800 CST m=+2.234391928 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:20:54.487 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:18:54.494 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:08:54.494 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/kv_client_stream_reconnect/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/kv_client_stream_reconnect/tiflash/log/error.log arg matches is ArgMatches { args: {"memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/kv_client_stream_reconnect/tiflash-proxy.toml"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/kv_client_stream_reconnect/tiflash/db/proxy"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/kv_client_stream_reconnect/tiflash/log/proxy.log"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489988012673080720 TiDB Cluster ID. tikv_gc_leader_uuid 65821dbca9c0005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-m-cm2pq, pid:1281, start at 2025-04-06 08:18:55.277903737 +0800 CST m=+2.453975236 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:20:55.286 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:18:55.271 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:08:55.271 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489988012673080720 TiDB Cluster ID. tikv_gc_leader_uuid 65821dbca9c0005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-m-cm2pq, pid:1281, start at 2025-04-06 08:18:55.277903737 +0800 CST m=+2.453975236 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:20:55.286 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:18:55.271 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:08:55.271 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489988010709352736 TiDB Cluster ID. tikv_gc_leader_uuid 65821dbc9f8000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-m-cm2pq, pid:1357, start at 2025-04-06 08:18:55.264919031 +0800 CST m=+2.377941877 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:20:55.273 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:18:55.267 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:08:55.267 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/sink_retry/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/sink_retry/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/sink_retry/tiflash/db/proxy"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/sink_retry/tiflash/log/proxy.log"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/sink_retry/tiflash-proxy.toml"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release start tidb cluster in /tmp/tidb_cdc_test/drop_many_tables Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489988006854039167 TiDB Cluster ID. tikv_gc_leader_uuid 65821dbcba8000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-c-m09xf, pid:1206, start at 2025-04-06 08:18:55.36058924 +0800 CST m=+2.331463243 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:20:55.370 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:18:55.338 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:08:55.338 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489988006854039167 TiDB Cluster ID. tikv_gc_leader_uuid 65821dbcba8000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-c-m09xf, pid:1206, start at 2025-04-06 08:18:55.36058924 +0800 CST m=+2.331463243 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:20:55.370 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:18:55.338 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:08:55.338 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489988012874932298 TiDB Cluster ID. tikv_gc_leader_uuid 65821dbcd48000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-c-m09xf, pid:1280, start at 2025-04-06 08:18:55.467833913 +0800 CST m=+2.335215729 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:20:55.477 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:18:55.442 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:08:55.442 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/foreign_key/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/foreign_key/tiflash/log/error.log arg matches is ArgMatches { args: {"log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/foreign_key/tiflash/log/proxy.log"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/foreign_key/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/foreign_key/tiflash/db/proxy"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Cache restored successfully (ws/jenkins-pingcap-tiflow-periodics_cdc_integration_mysql_test-234/tiflow-cdc) 4028702720 bytes in 6.63 secs (607221127 bytes/sec) [Pipeline] { [Pipeline] cache [Sun Apr 6 08:18:58 CST 2025] <<<<<< START cdc server in kv_client_stream_reconnect case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/kv/kvClientForceReconnect=return(true)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kv_client_stream_reconnect.26542656.out server --log-file /tmp/tidb_cdc_test/kv_client_stream_reconnect/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/kv_client_stream_reconnect/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489988012853061961 TiDB Cluster ID. tikv_gc_leader_uuid 65821dbe614000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-6-zhp59, pid:1211, start at 2025-04-06 08:18:57.073664838 +0800 CST m=+2.436069966 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:20:57.082 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:18:57.076 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:08:57.076 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489988012853061961 TiDB Cluster ID. tikv_gc_leader_uuid 65821dbe614000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-6-zhp59, pid:1211, start at 2025-04-06 08:18:57.073664838 +0800 CST m=+2.436069966 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:20:57.082 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:18:57.076 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:08:57.076 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489988019654940544 TiDB Cluster ID. tikv_gc_leader_uuid 65821dbe6b40007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-6-zhp59, pid:1284, start at 2025-04-06 08:18:57.080422779 +0800 CST m=+2.340811586 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:20:57.089 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:18:57.069 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:08:57.069 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489988021468440714 TiDB Cluster ID. tikv_gc_leader_uuid 65821dbe4ac000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-v-hm9ds, pid:1285, start at 2025-04-06 08:18:56.963843604 +0800 CST m=+2.718612495 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:20:56.973 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:18:57.025 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:08:57.025 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489988021468440714 TiDB Cluster ID. tikv_gc_leader_uuid 65821dbe4ac000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-v-hm9ds, pid:1285, start at 2025-04-06 08:18:56.963843604 +0800 CST m=+2.718612495 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:20:56.973 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:18:57.025 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:08:57.025 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489988016653489428 TiDB Cluster ID. tikv_gc_leader_uuid 65821dbe6340007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-v-hm9ds, pid:1342, start at 2025-04-06 08:18:57.051701159 +0800 CST m=+2.721774544 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:20:57.060 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:18:57.037 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:08:57.037 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/charset_gbk/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/charset_gbk/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/charset_gbk/tiflash/log/proxy.log"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/charset_gbk/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/charset_gbk/tiflash-proxy.toml"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/consistent_replicate_ddl/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/consistent_replicate_ddl/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/consistent_replicate_ddl/tiflash/db/proxy"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/consistent_replicate_ddl/tiflash-proxy.toml"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/consistent_replicate_ddl/tiflash/log/proxy.log"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.foreign_key.cli.2676.out cli tso query --pd=http://127.0.0.1:2379 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sink_retry.cli.2682.out cli tso query --pd=http://127.0.0.1:2379 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489988024264119541 TiDB Cluster ID. tikv_gc_leader_uuid 65821dbf214000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-d-7dnbd, pid:1290, start at 2025-04-06 08:18:57.820862914 +0800 CST m=+2.466487069 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:20:57.865 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:18:57.857 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:08:57.857 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489988024264119541 TiDB Cluster ID. tikv_gc_leader_uuid 65821dbf214000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-d-7dnbd, pid:1290, start at 2025-04-06 08:18:57.820862914 +0800 CST m=+2.466487069 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:20:57.865 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:18:57.857 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:08:57.857 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489988011583343352 TiDB Cluster ID. tikv_gc_leader_uuid 65821dbf2180006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-d-7dnbd, pid:1364, start at 2025-04-06 08:18:57.806546956 +0800 CST m=+2.393561409 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:20:57.814 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:18:57.857 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:08:57.857 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/processor_err_chan/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/processor_err_chan/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/processor_err_chan/tiflash/db/proxy"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/processor_err_chan/tiflash-proxy.toml"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/processor_err_chan/tiflash/log/proxy.log"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release [Sun Apr 6 08:19:00 CST 2025] <<<<<< START cdc server in charset_gbk case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.charset_gbk.26852687.out server --log-file /tmp/tidb_cdc_test/charset_gbk/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/charset_gbk/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + set +x + tso='457152591172993025 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 457152591172993025 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x ***************** properties ***************** "readallfields"="true" "dotransactions"="false" "requestdistribution"="uniform" "readproportion"="0" "threadcount"="2" "updateproportion"="0" "mysql.host"="127.0.0.1" "recordcount"="10" "scanproportion"="0" "insertproportion"="0" "mysql.port"="4000" "mysql.user"="root" "mysql.db"="sink_retry" "workload"="core" "operationcount"="0" ********************************************** Run finished, takes 8.66274ms INSERT - Takes(s): 0.0, Count: 10, OPS: 1917.6, Avg(us): 1646, Min(us): 929, Max(us): 3423, 95th(us): 4000, 99th(us): 4000 [Sun Apr 6 08:19:01 CST 2025] <<<<<< START cdc server in sink_retry case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sink/dmlsink/txn/mysql/MySQLSinkTxnRandomError=25%return(true)' + '[' -z '' ']' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sink_retry.27422744.out server --log-file /tmp/tidb_cdc_test/sink_retry/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/sink_retry/cdc_data --cluster-id default + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + set +x + tso='457152591230140418 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 457152591230140418 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Sun Apr 6 08:19:01 CST 2025] <<<<<< START cdc server in foreign_key case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.foreign_key.27082710.out server --log-file /tmp/tidb_cdc_test/foreign_key/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/foreign_key/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [Sun Apr 6 08:19:01 CST 2025] <<<<<< START cdc server in consistent_replicate_ddl case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.consistent_replicate_ddl.26702672.out server --log-file /tmp/tidb_cdc_test/consistent_replicate_ddl/cdcconsistent_replicate_ddl.server1.log --log-level debug --data-dir /tmp/tidb_cdc_test/consistent_replicate_ddl/cdc_dataconsistent_replicate_ddl.server1 --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 06 Apr 2025 00:19:01 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/32088003-28ef-4dcb-ba9c-145f2ccc4663 {"id":"32088003-28ef-4dcb-ba9c-145f2ccc4663","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898738} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/2231960876d85fd6 32088003-28ef-4dcb-ba9c-145f2ccc4663 /tidb/cdc/default/default/upstream/7489988010948606366 {"id":7489988010948606366,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/32088003-28ef-4dcb-ba9c-145f2ccc4663 {"id":"32088003-28ef-4dcb-ba9c-145f2ccc4663","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898738} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/2231960876d85fd6 32088003-28ef-4dcb-ba9c-145f2ccc4663 /tidb/cdc/default/default/upstream/7489988010948606366 {"id":7489988010948606366,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/32088003-28ef-4dcb-ba9c-145f2ccc4663 {"id":"32088003-28ef-4dcb-ba9c-145f2ccc4663","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898738} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/2231960876d85fd6 32088003-28ef-4dcb-ba9c-145f2ccc4663 /tidb/cdc/default/default/upstream/7489988010948606366 {"id":7489988010948606366,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489988041711609721 TiDB Cluster ID. tikv_gc_leader_uuid 65821dc3120000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-8-31x9h, pid:1389, start at 2025-04-06 08:19:01.84842944 +0800 CST m=+2.341035836 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:21:01.858 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:19:01.832 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:09:01.832 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489988041711609721 TiDB Cluster ID. tikv_gc_leader_uuid 65821dc3120000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-8-31x9h, pid:1389, start at 2025-04-06 08:19:01.84842944 +0800 CST m=+2.341035836 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:21:01.858 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:19:01.832 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:09:01.832 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489988028918415635 TiDB Cluster ID. tikv_gc_leader_uuid 65821dc2f84000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-8-31x9h, pid:1454, start at 2025-04-06 08:19:01.756061547 +0800 CST m=+2.190376545 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:21:01.764 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:19:01.729 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:09:01.729 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/http_api/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/http_api/tiflash/log/error.log arg matches is ArgMatches { args: {"pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/http_api/tiflash/log/proxy.log"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/http_api/tiflash/db/proxy"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/http_api/tiflash-proxy.toml"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 06 Apr 2025 00:19:03 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7b15eb38-bc20-45d4-8538-937538b37244 {"id":"7b15eb38-bc20-45d4-8538-937538b37244","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898741} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/2231960876dca0d9 7b15eb38-bc20-45d4-8538-937538b37244 /tidb/cdc/default/default/upstream/7489988021468440714 {"id":7489988021468440714,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7b15eb38-bc20-45d4-8538-937538b37244 {"id":"7b15eb38-bc20-45d4-8538-937538b37244","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898741} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/2231960876dca0d9 7b15eb38-bc20-45d4-8538-937538b37244 /tidb/cdc/default/default/upstream/7489988021468440714 {"id":7489988021468440714,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7b15eb38-bc20-45d4-8538-937538b37244 {"id":"7b15eb38-bc20-45d4-8538-937538b37244","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898741} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/2231960876dca0d9 7b15eb38-bc20-45d4-8538-937538b37244 /tidb/cdc/default/default/upstream/7489988021468440714 {"id":7489988021468440714,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: 347fb3b4-cccb-4f31-ab7f-f099764cb439 Info: {"upstream_id":7489988021468440714,"namespace":"default","id":"347fb3b4-cccb-4f31-ab7f-f099764cb439","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-06T08:19:04.078414457+08:00","start_ts":457152591505391617,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457152591505391617,"checkpoint_ts":457152591505391617,"checkpoint_time":"2025-04-06 08:19:00.789"} + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 06 Apr 2025 00:19:04 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b1b015e7-f3d6-4097-9b22-e0925ed582fd {"id":"b1b015e7-f3d6-4097-9b22-e0925ed582fd","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898741} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/2231960876d909ef b1b015e7-f3d6-4097-9b22-e0925ed582fd /tidb/cdc/default/default/upstream/7489988012673080720 {"id":7489988012673080720,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b1b015e7-f3d6-4097-9b22-e0925ed582fd {"id":"b1b015e7-f3d6-4097-9b22-e0925ed582fd","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898741} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/2231960876d909ef b1b015e7-f3d6-4097-9b22-e0925ed582fd /tidb/cdc/default/default/upstream/7489988012673080720 {"id":7489988012673080720,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b1b015e7-f3d6-4097-9b22-e0925ed582fd {"id":"b1b015e7-f3d6-4097-9b22-e0925ed582fd","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898741} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/2231960876d909ef b1b015e7-f3d6-4097-9b22-e0925ed582fd /tidb/cdc/default/default/upstream/7489988012673080720 {"id":7489988012673080720,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sink_retry.cli.2804.out cli changefeed create --start-ts=457152591172993025 '--sink-uri=mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1' ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 06 Apr 2025 00:19:04 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3d748c0e-05dc-4b32-996c-2a44f0b0413d {"id":"3d748c0e-05dc-4b32-996c-2a44f0b0413d","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898741} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/2231960876d5c9e9 3d748c0e-05dc-4b32-996c-2a44f0b0413d /tidb/cdc/default/default/upstream/7489988006854039167 {"id":7489988006854039167,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3d748c0e-05dc-4b32-996c-2a44f0b0413d {"id":"3d748c0e-05dc-4b32-996c-2a44f0b0413d","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898741} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/2231960876d5c9e9 3d748c0e-05dc-4b32-996c-2a44f0b0413d /tidb/cdc/default/default/upstream/7489988006854039167 {"id":7489988006854039167,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3d748c0e-05dc-4b32-996c-2a44f0b0413d {"id":"3d748c0e-05dc-4b32-996c-2a44f0b0413d","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898741} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/2231960876d5c9e9 3d748c0e-05dc-4b32-996c-2a44f0b0413d /tidb/cdc/default/default/upstream/7489988006854039167 {"id":7489988006854039167,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.foreign_key.cli.2773.out cli changefeed create --start-ts=457152591230140418 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ --config=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/foreign_key/conf/cf.toml Create changefeed successfully! ID: 40b91ab0-9636-49ee-97b6-b413a01156e7 Info: {"upstream_id":7489988012673080720,"namespace":"default","id":"40b91ab0-9636-49ee-97b6-b413a01156e7","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create_time":"2025-04-06T08:19:04.589672133+08:00","start_ts":457152591172993025,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457152591172993025,"checkpoint_ts":457152591172993025,"checkpoint_time":"2025-04-06 08:18:59.521"} PASS + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 06 Apr 2025 00:19:04 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7137b869-78e6-418f-a64a-bcffff2fc719 {"id":"7137b869-78e6-418f-a64a-bcffff2fc719","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898741} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/2231960876da0adc 7137b869-78e6-418f-a64a-bcffff2fc719 /tidb/cdc/default/default/upstream/7489988012853061961 {"id":7489988012853061961,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7137b869-78e6-418f-a64a-bcffff2fc719 {"id":"7137b869-78e6-418f-a64a-bcffff2fc719","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898741} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/2231960876da0adc 7137b869-78e6-418f-a64a-bcffff2fc719 /tidb/cdc/default/default/upstream/7489988012853061961 {"id":7489988012853061961,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7137b869-78e6-418f-a64a-bcffff2fc719 {"id":"7137b869-78e6-418f-a64a-bcffff2fc719","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898741} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/2231960876da0adc 7137b869-78e6-418f-a64a-bcffff2fc719 /tidb/cdc/default/default/upstream/7489988012853061961 {"id":7489988012853061961,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: 4e34db21-9bd2-4739-9806-43bab24ad00a Info: {"upstream_id":7489988006854039167,"namespace":"default","id":"4e34db21-9bd2-4739-9806-43bab24ad00a","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-06T08:19:04.772175506+08:00","start_ts":457152591230140418,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*","!foreign_key.t12"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457152591230140418,"checkpoint_ts":457152591230140418,"checkpoint_time":"2025-04-06 08:18:59.739"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... coverage: 2.4% of statements in github.com/pingcap/tiflow/... ***************** properties ***************** "insertproportion"="0" "operationcount"="0" "mysql.user"="root" "threadcount"="10" "dotransactions"="false" "updateproportion"="0" "mysql.port"="4000" "requestdistribution"="uniform" "readallfields"="true" "scanproportion"="0" "readproportion"="0" "mysql.host"="127.0.0.1" "recordcount"="50" "workload"="core" "mysql.db"="consistent_replicate_ddl" "fieldcount"="3" ********************************************** Run finished, takes 12.053044ms INSERT - Takes(s): 0.0, Count: 50, OPS: 7654.8, Avg(us): 2199, Min(us): 1052, Max(us): 5567, 95th(us): 6000, 99th(us): 6000 + set +x + set +x VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489988051423773347 TiDB Cluster ID. tikv_gc_leader_uuid 65821dc59640006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-k-bx9s4, pid:3989, start at 2025-04-06 08:19:04.418011544 +0800 CST m=+2.555522940 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:21:04.461 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:19:04.459 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:09:04.459 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489988051423773347 TiDB Cluster ID. tikv_gc_leader_uuid 65821dc59640006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-k-bx9s4, pid:3989, start at 2025-04-06 08:19:04.418011544 +0800 CST m=+2.555522940 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:21:04.461 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:19:04.459 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:09:04.459 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489988052832033977 TiDB Cluster ID. tikv_gc_leader_uuid 65821dc5894000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-k-bx9s4, pid:4064, start at 2025-04-06 08:19:04.398889226 +0800 CST m=+2.478990547 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:21:04.410 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:19:04.404 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:09:04.404 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/drop_many_tables/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/drop_many_tables/tiflash/log/error.log arg matches is ArgMatches { args: {"log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/drop_many_tables/tiflash/log/proxy.log"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/drop_many_tables/tiflash-proxy.toml"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/drop_many_tables/tiflash/db/proxy"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [Sun Apr 6 08:19:05 CST 2025] <<<<<< START cdc server in http_api case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.http_api.27982800.out server --log-file /tmp/tidb_cdc_test/http_api/cdc1.log --log-level debug --data-dir /tmp/tidb_cdc_test/http_api/cdc_data1 --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table foreign_key.finish_mark not exists for 1-th check, retry later table consistent_replicate_ddl.usertable1 not exists for 1-th check, retry later Cache restored successfully (ws/jenkins-pingcap-tiflow-periodics_cdc_integration_mysql_test-234/tiflow-cdc) 4028702720 bytes in 7.89 secs (510752854 bytes/sec) [Pipeline] { [Pipeline] cache + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.drop_many_tables.cli.5372.out cli tso query --pd=http://127.0.0.1:2379 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 06 Apr 2025 00:19:08 GMT < Content-Length: 864 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/facef94b-31cb-4cfa-b41a-7a279a59cc56 {"id":"facef94b-31cb-4cfa-b41a-7a279a59cc56","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898746} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/2231960876f30be0 facef94b-31cb-4cfa-b41a-7a279a59cc56 /tidb/cdc/default/default/upstream/7489988041711609721 {"id":7489988041711609721,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/facef94b-31cb-4cfa-b41a-7a279a59cc56 {"id":"facef94b-31cb-4cfa-b41a-7a279a59cc56","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898746} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/2231960876f30be0 facef94b-31cb-4cfa-b41a-7a279a59cc56 /tidb/cdc/default/default/upstream/7489988041711609721 {"id":7489988041711609721,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/facef94b-31cb-4cfa-b41a-7a279a59cc56 {"id":"facef94b-31cb-4cfa-b41a-7a279a59cc56","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898746} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/2231960876f30be0 facef94b-31cb-4cfa-b41a-7a279a59cc56 /tidb/cdc/default/default/upstream/7489988041711609721 {"id":7489988041711609721,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x cdc.test cli capture list 2>&1 | grep '"is-owner": true' table foreign_key.finish_mark not exists for 2-th check, retry later table tidb_cdc.syncpoint_v1 exists table tidb_cdc.syncpoint_v1 exists table bdr_mode.finish_mark exists table bdr_mode.finish_mark not exists for 1-th check, retry later table bdr_mode.finish_mark exists check diff successfully [Sun Apr 6 08:19:08 CST 2025] <<<<<< START cdc server in processor_err_chan case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/processor/ProcessorAddTableError=1*return(true)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.processor_err_chan.28302832.out server --log-file /tmp/tidb_cdc_test/processor_err_chan/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/processor_err_chan/cdc_data --cluster-id default --config /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/processor_err_chan/conf/server.toml --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 wait process cdc.test exit for 1-th time... table consistent_replicate_ddl.usertable1 exists table consistent_replicate_ddl.usertable2 not exists for 1-th check, retry later + set +x + tso='457152593555357697 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 457152593555357697 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Sun Apr 6 08:19:10 CST 2025] <<<<<< START cdc server in drop_many_tables case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.drop_many_tables.54045406.out server --log-file /tmp/tidb_cdc_test/drop_many_tables/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/drop_many_tables/cdc_data --cluster-id default + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 "is-owner": true, run task successfully wait process cdc.test exit for 2-th time... table charset_gbk_test0.t0 exists table charset_gbk_test0.t1 exists table charset_gbk_test1.t0 not exists for 1-th check, retry later table foreign_key.finish_mark not exists for 3-th check, retry later wait process cdc.test exit for 3-th time... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 06 Apr 2025 00:19:11 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/fa8f47d0-c203-4dd4-8935-3f79ebb9631f {"id":"fa8f47d0-c203-4dd4-8935-3f79ebb9631f","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898748} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/2231960876db1a61 fa8f47d0-c203-4dd4-8935-3f79ebb9631f /tidb/cdc/default/default/upstream/7489988024264119541 {"id":7489988024264119541,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/fa8f47d0-c203-4dd4-8935-3f79ebb9631f {"id":"fa8f47d0-c203-4dd4-8935-3f79ebb9631f","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898748} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/2231960876db1a61 fa8f47d0-c203-4dd4-8935-3f79ebb9631f /tidb/cdc/default/default/upstream/7489988024264119541 {"id":7489988024264119541,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/fa8f47d0-c203-4dd4-8935-3f79ebb9631f {"id":"fa8f47d0-c203-4dd4-8935-3f79ebb9631f","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898748} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/2231960876db1a61 fa8f47d0-c203-4dd4-8935-3f79ebb9631f /tidb/cdc/default/default/upstream/7489988024264119541 {"id":7489988024264119541,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x check_changefeed_state http://127.0.0.1:2379 40161fe8-4179-4b76-a095-e10b2d401cae normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=40161fe8-4179-4b76-a095-e10b2d401cae + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c 40161fe8-4179-4b76-a095-e10b2d401cae -s cdc.test: no process found + info='{ "upstream_id": 7489988024264119541, wait process cdc.test exit for 4-th time... process cdc.test already exit [Sun Apr 6 08:19:11 CST 2025] <<<<<< run test case bdr_mode success! >>>>>> "namespace": "default", "id": "40161fe8-4179-4b76-a095-e10b2d401cae", "state": "normal", "checkpoint_tso": 457152594247155714, "checkpoint_time": "2025-04-06 08:19:11.248", "error": null }' + echo '{ "upstream_id": 7489988024264119541, "namespace": "default", "id": "40161fe8-4179-4b76-a095-e10b2d401cae", "state": "normal", "checkpoint_tso": 457152594247155714, "checkpoint_time": "2025-04-06 08:19:11.248", "error": null }' { "upstream_id": 7489988024264119541, "namespace": "default", "id": "40161fe8-4179-4b76-a095-e10b2d401cae", "state": "normal", "checkpoint_tso": 457152594247155714, "checkpoint_time": "2025-04-06 08:19:11.248", "error": null } ++ echo '{' '"upstream_id":' 7489988024264119541, '"namespace":' '"default",' '"id":' '"40161fe8-4179-4b76-a095-e10b2d401cae",' '"state":' '"normal",' '"checkpoint_tso":' 457152594247155714, '"checkpoint_time":' '"2025-04-06' '08:19:11.248",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7489988024264119541, '"namespace":' '"default",' '"id":' '"40161fe8-4179-4b76-a095-e10b2d401cae",' '"state":' '"normal",' '"checkpoint_tso":' 457152594247155714, '"checkpoint_time":' '"2025-04-06' '08:19:11.248",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully table consistent_replicate_ddl.usertable2 not exists for 2-th check, retry later owner pid: 2803 owner id facef94b-31cb-4cfa-b41a-7a279a59cc56 default [Sun Apr 6 08:19:11 CST 2025] <<<<<< START cdc server in http_api case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.http_api.29192921.out server --log-file /tmp/tidb_cdc_test/http_api/cdc2.log --log-level debug --data-dir /tmp/tidb_cdc_test/http_api/cdc_data2 --cluster-id default --addr 127.0.0.1:8301 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table charset_gbk_test1.t0 not exists for 2-th check, retry later check diff failed 1-th time, retry later table foreign_key.finish_mark not exists for 4-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 06 Apr 2025 00:19:13 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c52b8366-55da-4d5b-9ab1-a58335e01d55 {"id":"c52b8366-55da-4d5b-9ab1-a58335e01d55","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898750} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/2231960876fa63e3 c52b8366-55da-4d5b-9ab1-a58335e01d55 /tidb/cdc/default/default/upstream/7489988051423773347 {"id":7489988051423773347,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c52b8366-55da-4d5b-9ab1-a58335e01d55 {"id":"c52b8366-55da-4d5b-9ab1-a58335e01d55","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898750} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/2231960876fa63e3 c52b8366-55da-4d5b-9ab1-a58335e01d55 /tidb/cdc/default/default/upstream/7489988051423773347 {"id":7489988051423773347,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c52b8366-55da-4d5b-9ab1-a58335e01d55 {"id":"c52b8366-55da-4d5b-9ab1-a58335e01d55","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898750} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/2231960876fa63e3 c52b8366-55da-4d5b-9ab1-a58335e01d55 /tidb/cdc/default/default/upstream/7489988051423773347 {"id":7489988051423773347,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.drop_many_tables.cli.5461.out cli changefeed create --start-ts=457152593555357697 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ Create changefeed successfully! ID: 0b82c736-9f8a-4415-b08d-ec50af667643 Info: {"upstream_id":7489988051423773347,"namespace":"default","id":"0b82c736-9f8a-4415-b08d-ec50af667643","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-06T08:19:13.549067588+08:00","start_ts":457152593555357697,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457152593555357697,"checkpoint_ts":457152593555357697,"checkpoint_time":"2025-04-06 08:19:08.609"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... table consistent_replicate_ddl.usertable2 exists table consistent_replicate_ddl.usertable3 not exists for 1-th check, retry later table charset_gbk_test1.t0 exists table test.finish_mark not exists for 1-th check, retry later check diff failed 2-th time, retry later + set +x table foreign_key.finish_mark not exists for 5-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 06 Apr 2025 00:19:14 GMT < Content-Length: 1325 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/898d0c68-f9f7-4d3e-9933-6a32070d1d1c {"id":"898d0c68-f9f7-4d3e-9933-6a32070d1d1c","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898752} /tidb/cdc/default/__cdc_meta__/capture/facef94b-31cb-4cfa-b41a-7a279a59cc56 {"id":"facef94b-31cb-4cfa-b41a-7a279a59cc56","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898746} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/2231960876f30be0 facef94b-31cb-4cfa-b41a-7a279a59cc56 /tidb/cdc/default/__cdc_meta__/owner/2231960876f30c12 898d0c68-f9f7-4d3e-9933-6a32070d1d1c /tidb/cdc/default/default/upstream/7489988041711609721 {"id":7489988041711609721,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/898d0c68-f9f7-4d3e-9933-6a32070d1d1c {"id":"898d0c68-f9f7-4d3e-9933-6a32070d1d1c","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898752} /tidb/cdc/default/__cdc_meta__/capture/facef94b-31cb-4cfa-b41a-7a279a59cc56 {"id":"facef94b-31cb-4cfa-b41a-7a279a59cc56","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898746} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/2231960876f30be0 facef94b-31cb-4cfa-b41a-7a279a59cc56 /tidb/cdc/default/__cdc_meta__/owner/2231960876f30c12 898d0c68-f9f7-4d3e-9933-6a32070d1d1c /tidb/cdc/default/default/upstream/7489988041711609721 {"id":7489988041711609721,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/898d0c68-f9f7-4d3e-9933-6a32070d1d1c {"id":"898d0c68-f9f7-4d3e-9933-6a32070d1d1c","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898752} /tidb/cdc/default/__cdc_meta__/capture/facef94b-31cb-4cfa-b41a-7a279a59cc56 {"id":"facef94b-31cb-4cfa-b41a-7a279a59cc56","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898746} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/2231960876f30be0 facef94b-31cb-4cfa-b41a-7a279a59cc56 /tidb/cdc/default/__cdc_meta__/owner/2231960876f30c12 898d0c68-f9f7-4d3e-9933-6a32070d1d1c /tidb/cdc/default/default/upstream/7489988041711609721 {"id":7489988041711609721,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x cdc.test cli capture list 2>&1 | grep -v "facef94b-31cb-4cfa-b41a-7a279a59cc56 default" | grep id table consistent_replicate_ddl.usertable3 not exists for 2-th check, retry later table test.finish_mark not exists for 2-th check, retry later "id": "898d0c68-f9f7-4d3e-9933-6a32070d1d1c", run task successfully check diff successfully table foreign_key.finish_mark not exists for 6-th check, retry later wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... table consistent_replicate_ddl.usertable3 exists table consistent_replicate_ddl.usertable_bak not exists for 1-th check, retry later capture_id: 898d0c68-f9f7-4d3e-9933-6a32070d1d1c Cache restored successfully (ws/jenkins-pingcap-tiflow-periodics_cdc_integration_mysql_test-234/tiflow-cdc) 4028702720 bytes in 7.45 secs (540810889 bytes/sec) [Pipeline] { [Pipeline] cache pass test: check health pass test: get status cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Sun Apr 6 08:19:17 CST 2025] <<<<<< run test case processor_err_chan success! >>>>>> table test.finish_mark not exists for 3-th check, retry later pass test: create changefeed table foreign_key.finish_mark not exists for 7-th check, retry later table drop_tables.c not exists for 1-th check, retry later table consistent_replicate_ddl.usertable_bak not exists for 2-th check, retry later table test.finish_mark not exists for 4-th check, retry later table foreign_key.finish_mark not exists for 8-th check, retry later table drop_tables.c not exists for 2-th check, retry later table consistent_replicate_ddl.usertable_bak exists table test.finish_mark exists check table exists success check diff successfully wait process cdc.test exit for 1-th time... table test.simple1 not exists for 1-th check, retry later wait process cdc.test exit for 2-th time... table foreign_key.finish_mark not exists for 9-th check, retry later cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Sun Apr 6 08:19:23 CST 2025] <<<<<< run test case charset_gbk success! >>>>>> table drop_tables.c not exists for 3-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/capture_suicide_while_balance_table/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... table test.simple1 exists table foreign_key.finish_mark not exists for 10-th check, retry later pass test: list changefeed table drop_tables.c exists check diff successfully pass test: get changefeed wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Sun Apr 6 08:19:26 CST 2025] <<<<<< run test case drop_many_tables success! >>>>>> start tidb cluster in /tmp/tidb_cdc_test/capture_suicide_while_balance_table Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... table foreign_key.finish_mark not exists for 11-th check, retry later wait process cdc.test exit for 1-th time... pass test: pause changefeed pass test: update changefeed wait process cdc.test exit for 2-th time... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/changefeed_reconstruct/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... wait process 2803 exit for 1-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Sun Apr 6 08:19:28 CST 2025] <<<<<< START cdc server in consistent_replicate_ddl case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sink/dmlsink/txn/mysql/MySQLSinkHangLongTime=return(true);github.com/pingcap/tiflow/cdc/sink/ddlsink/mysql/MySQLSinkExecDDLDelay=return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.consistent_replicate_ddl.29072909.out server --log-file /tmp/tidb_cdc_test/consistent_replicate_ddl/cdcconsistent_replicate_ddl.server2.log --log-level debug --data-dir /tmp/tidb_cdc_test/consistent_replicate_ddl/cdc_dataconsistent_replicate_ddl.server2 --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 wait process 2803 exit for 2-th time... table foreign_key.finish_mark not exists for 12-th check, retry later /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (2803) - No such process wait process 2803 exit for 3-th time... process 2803 already exit cdc.test cli capture list --server http://127.0.0.1:8301 2>&1 |grep 898d0c68-f9f7-4d3e-9933-6a32070d1d1c -A1 | grep '"is-owner": true' Cache restored successfully (ws/jenkins-pingcap-tiflow-periodics_cdc_integration_mysql_test-234/tiflow-cdc) 4028702720 bytes in 10.18 secs (395592846 bytes/sec) [Pipeline] { [Pipeline] cache Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release "is-owner": true, run task successfully [Sun Apr 6 08:19:30 CST 2025] <<<<<< START cdc server in http_api case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.http_api.31983200.out server --log-file /tmp/tidb_cdc_test/http_api/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/http_api/cdc_data --cluster-id default + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table foreign_key.finish_mark not exists for 13-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/changefeed_reconstruct Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 06 Apr 2025 00:19:31 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/f7a08292-07f7-4b1c-a2df-56781647ce79 {UpstreamID:7489988012853061961 Namespace:default ID:f7a08292-07f7-4b1c-a2df-56781647ce79 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-06 08:19:04.316903106 +0800 CST StartTs:457152592406380549 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0037c5b90 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457152592419225608} {CheckpointTs:457152597505081347 MinTableBarrierTs:457152599339827205 AdminJobType:noop} span: {tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}, resolvedTs: 457152597505081347, checkpointTs: 457152597505081347, state: Replicating span: {tableID:116, startKey:7480000000000000ff745f720000000000fa, endKey:7480000000000000ff745f730000000000fa}, resolvedTs: 457152597505081347, checkpointTs: 457152597505081347, state: Replicating span: {tableID:118, startKey:7480000000000000ff765f720000000000fa, endKey:7480000000000000ff765f730000000000fa}, resolvedTs: 457152597505081347, checkpointTs: 457152597505081347, state: Replicating span: {tableID:120, startKey:7480000000000000ff785f720000000000fa, endKey:7480000000000000ff785f730000000000fa}, resolvedTs: 457152597505081347, checkpointTs: 457152597505081347, state: Replicating span: {tableID:122, startKey:7480000000000000ff7a5f720000000000fa, endKey:7480000000000000ff7a5f730000000000fa}, resolvedTs: 457152597505081347, checkpointTs: 457152597505081347, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/56481004-231f-47b9-bb92-69fccc27f9f1 {"id":"56481004-231f-47b9-bb92-69fccc27f9f1","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898768} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/2231960876da0c21 56481004-231f-47b9-bb92-69fccc27f9f1 /tidb/cdc/default/default/changefeed/info/f7a08292-07f7-4b1c-a2df-56781647ce79 {"upstream-id":7489988012853061961,"namespace":"default","changefeed-id":"f7a08292-07f7-4b1c-a2df-56781647ce79","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-06T08:19:04.316903106+08:00","start-ts":457152592406380549,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"file:///tmp/tidb_cdc_test/consistent_replicate_ddl/redo","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457152592419225608} /tidb/cdc/default/default/changefeed/status/f7a08292-07f7-4b1c-a2df-56781647ce79 {"checkpoint-ts":457152597505081347,"min-table-barrier-ts":457152599339827205,"admin-job-type":0} /tidb/cdc/default/default/task/position/56481004-231f-47b9-bb92-69fccc27f9f1/f7a08292-07f7-4b1c-a2df-56781647ce79 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489988012853061961 {"id":7489988012853061961,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/f7a08292-07f7-4b1c-a2df-56781647ce79 {UpstreamID:7489988012853061961 Namespace:default ID:f7a08292-07f7-4b1c-a2df-56781647ce79 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-06 08:19:04.316903106 +0800 CST StartTs:457152592406380549 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0037c5b90 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457152592419225608} {CheckpointTs:457152597505081347 MinTableBarrierTs:457152599339827205 AdminJobType:noop} span: {tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}, resolvedTs: 457152597505081347, checkpointTs: 457152597505081347, state: Replicating span: {tableID:116, startKey:7480000000000000ff745f720000000000fa, endKey:7480000000000000ff745f730000000000fa}, resolvedTs: 457152597505081347, checkpointTs: 457152597505081347, state: Replicating span: {tableID:118, startKey:7480000000000000ff765f720000000000fa, endKey:7480000000000000ff765f730000000000fa}, resolvedTs: 457152597505081347, checkpointTs: 457152597505081347, state: Replicating span: {tableID:120, startKey:7480000000000000ff785f720000000000fa, endKey:7480000000000000ff785f730000000000fa}, resolvedTs: 457152597505081347, checkpointTs: 457152597505081347, state: Replicating span: {tableID:122, startKey:7480000000000000ff7a5f720000000000fa, endKey:7480000000000000ff7a5f730000000000fa}, resolvedTs: 457152597505081347, checkpointTs: 457152597505081347, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/56481004-231f-47b9-bb92-69fccc27f9f1 {"id":"56481004-231f-47b9-bb92-69fccc27f9f1","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898768} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/2231960876da0c21 56481004-231f-47b9-bb92-69fccc27f9f1 /tidb/cdc/default/default/changefeed/info/f7a08292-07f7-4b1c-a2df-56781647ce79 {"upstream-id":7489988012853061961,"namespace":"default","changefeed-id":"f7a08292-07f7-4b1c-a2df-56781647ce79","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-06T08:19:04.316903106+08:00","start-ts":457152592406380549,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"file:///tmp/tidb_cdc_test/consistent_replicate_ddl/redo","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457152592419225608} /tidb/cdc/default/default/changefeed/status/f7a08292-07f7-4b1c-a2df-56781647ce79 {"checkpoint-ts":457152597505081347,"min-table-barrier-ts":457152599339827205,"admin-job-type":0} /tidb/cdc/default/default/task/position/56481004-231f-47b9-bb92-69fccc27f9f1/f7a08292-07f7-4b1c-a2df-56781647ce79 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489988012853061961 {"id":7489988012853061961,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/f7a08292-07f7-4b1c-a2df-56781647ce79 {UpstreamID:7489988012853061961 Namespace:default ID:f7a08292-07f7-4b1c-a2df-56781647ce79 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-06 08:19:04.316903106 +0800 CST StartTs:457152592406380549 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0037c5b90 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457152592419225608} {CheckpointTs:457152597505081347 MinTableBarrierTs:457152599339827205 AdminJobType:noop} span: {tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}, resolvedTs: 457152597505081347, checkpointTs: 457152597505081347, state: Replicating span: {tableID:116, startKey:7480000000000000ff745f720000000000fa, endKey:7480000000000000ff745f730000000000fa}, resolvedTs: 457152597505081347, checkpointTs: 457152597505081347, state: Replicating span: {tableID:118, startKey:7480000000000000ff765f720000000000fa, endKey:7480000000000000ff765f730000000000fa}, resolvedTs: 457152597505081347, checkpointTs: 457152597505081347, state: Replicating span: {tableID:120, startKey:7480000000000000ff785f720000000000fa, endKey:7480000000000000ff785f730000000000fa}, resolvedTs: 457152597505081347, checkpointTs: 457152597505081347, state: Replicating span: {tableID:122, startKey:7480000000000000ff7a5f720000000000fa, endKey:7480000000000000ff7a5f730000000000fa}, resolvedTs: 457152597505081347, checkpointTs: 457152597505081347, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/56481004-231f-47b9-bb92-69fccc27f9f1 {"id":"56481004-231f-47b9-bb92-69fccc27f9f1","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898768} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/2231960876da0c21 56481004-231f-47b9-bb92-69fccc27f9f1 /tidb/cdc/default/default/changefeed/info/f7a08292-07f7-4b1c-a2df-56781647ce79 {"upstream-id":7489988012853061961,"namespace":"default","changefeed-id":"f7a08292-07f7-4b1c-a2df-56781647ce79","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-06T08:19:04.316903106+08:00","start-ts":457152592406380549,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"file:///tmp/tidb_cdc_test/consistent_replicate_ddl/redo","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457152592419225608} /tidb/cdc/default/default/changefeed/status/f7a08292-07f7-4b1c-a2df-56781647ce79 {"checkpoint-ts":457152597505081347,"min-table-barrier-ts":457152599339827205,"admin-job-type":0} /tidb/cdc/default/default/task/position/56481004-231f-47b9-bb92-69fccc27f9f1/f7a08292-07f7-4b1c-a2df-56781647ce79 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489988012853061961 {"id":7489988012853061961,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table foreign_key.finish_mark exists check diff successfully table foreign_key.t12 does not exists wait process cdc.test exit for 1-th time... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff successfully wait process cdc.test exit for 1-th time... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 06 Apr 2025 00:19:33 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/changefeed-test3 {UpstreamID:7489988041711609721 Namespace:default ID:changefeed-test3 SinkURI:blackhole:// CreateTime:2025-04-06 08:19:18.475899081 +0800 CST StartTs:457152596104183818 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003932fc0 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457152596130398218} {CheckpointTs:457152596602257438 MinTableBarrierTs:457152599839735813 AdminJobType:noop} span: {tableID:116, startKey:7480000000000000ff745f720000000000fa, endKey:7480000000000000ff745f730000000000fa}, resolvedTs: 457152596602257438, checkpointTs: 457152596602257438, state: Preparing changefeedID: default/changefeed-test1 {UpstreamID:7489988041711609721 Namespace:default ID:changefeed-test1 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-06 08:19:18.176200435 +0800 CST StartTs:457152596038909964 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc004a2a000 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457152596051755019} {CheckpointTs:457152596602257438 MinTableBarrierTs:457152599839735813 AdminJobType:noop} span: {tableID:112, startKey:7480000000000000ff705f720000000000fa, endKey:7480000000000000ff705f730000000000fa}, resolvedTs: 457152596602257438, checkpointTs: 457152596602257438, state: Preparing span: {tableID:116, startKey:7480000000000000ff745f720000000000fa, endKey:7480000000000000ff745f730000000000fa}, resolvedTs: 457152596602257438, checkpointTs: 457152596602257438, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/066c2fc4-2313-4926-9ee4-ee0a8ecfed16 {"id":"066c2fc4-2313-4926-9ee4-ee0a8ecfed16","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898770} /tidb/cdc/default/__cdc_meta__/capture/898d0c68-f9f7-4d3e-9933-6a32070d1d1c {"id":"898d0c68-f9f7-4d3e-9933-6a32070d1d1c","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898752} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 1 /tidb/cdc/default/__cdc_meta__/owner/2231960876f30c12 898d0c68-f9f7-4d3e-9933-6a32070d1d1c /tidb/cdc/default/__cdc_meta__/owner/2231960876f30ca6 066c2fc4-2313-4926-9ee4-ee0a8ecfed16 /tidb/cdc/default/default/changefeed/info/changefeed-test1 {"upstream-id":7489988041711609721,"namespace":"default","changefeed-id":"changefeed-test1","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-06T08:19:18.176200435+08:00","start-ts":457152596038909964,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457152596051755019} /tidb/cdc/default/default/changefeed/info/changefeed-test2 {"upstream-id":7489988041711609721,"namespace":"default","changefeed-id":"changefeed-test2","sink-uri":"blackhole://","create-time":"2025-04-06T08:19:18.292255943+08:00","start-ts":457152596078231562,"target-ts":0,"admin-job-type":1,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":32},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"stopped","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457152598083371011} /tidb/cdc/default/default/changefeed/info/changefeed-test3 {"upstream-id":7489988041711609721,"namespace":"default","changefeed-id":"changefeed-test3","sink-uri":"blackhole://","create-time":"2025-04-06T08:19:18.475899081+08:00","start-ts":457152596104183818,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457152596130398218} /tidb/cdc/default/default/changefeed/status/changefeed-test1 {"checkpoint-ts":457152596602257438,"min-table-barrier-ts":457152599839735813,"admin-job-type":0} /tidb/cdc/default/default/changefeed/status/changefeed-test2 {"checkpoint-ts":457152596602257438,"min-table-barrier-ts":457152596602257438,"admin-job-type":1} /tidb/cdc/default/default/changefeed/status/changefeed-test3 {"checkpoint-ts":457152596602257438,"min-table-barrier-ts":457152599839735813,"admin-job-type":0} /tidb/cdc/default/default/task/position/066c2fc4-2313-4926-9ee4-ee0a8ecfed16/changefeed-test1 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/066c2fc4-2313-4926-9ee4-ee0a8ecfed16/changefeed-test3 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/898d0c68-f9f7-4d3e-9933-6a32070d1d1c/changefeed-test1 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/898d0c68-f9f7-4d3e-9933-6a32070d1d1c/changefeed-test3 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489988041711609721 {"id":7489988041711609721,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: changefeedID: default/changefeed-test3 {UpstreamID:7489988041711609721 Namespace:default ID:changefeed-test3 SinkURI:blackhole:// CreateTime:2025-04-06 08:19:18.475899081 +0800 CST StartTs:457152596104183818 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003932fc0 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457152596130398218} {CheckpointTs:457152596602257438 MinTableBarrierTs:457152599839735813 AdminJobType:noop} span: {tableID:116, startKey:7480000000000000ff745f720000000000fa, endKey:7480000000000000ff745f730000000000fa}, resolvedTs: 457152596602257438, checkpointTs: 457152596602257438, state: Preparing changefeedID: default/changefeed-test1 {UpstreamID:7489988041711609721 Namespace:default ID:changefeed-test1 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-06 08:19:18.176200435 +0800 CST StartTs:457152596038909964 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc004a2a000 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457152596051755019} {CheckpointTs:457152596602257438 MinTableBarrierTs:457152599839735813 AdminJobType:noop} span: {tableID:112, startKey:7480000000000000ff705f720000000000fa, endKey:7480000000000000ff705f730000000000fa}, resolvedTs: 457152596602257438, checkpointTs: 457152596602257438, state: Preparing span: {tableID:116, startKey:7480000000000000ff745f720000000000fa, endKey:7480000000000000ff745f730000000000fa}, resolvedTs: 457152596602257438, checkpointTs: 457152596602257438, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/066c2fc4-2313-4926-9ee4-ee0a8ecfed16 {"id":"066c2fc4-2313-4926-9ee4-ee0a8ecfed16","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898770} /tidb/cdc/default/__cdc_meta__/capture/898d0c68-f9f7-4d3e-9933-6a32070d1d1c {"id":"898d0c68-f9f7-4d3e-9933-6a32070d1d1c","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898752} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 1 /tidb/cdc/default/__cdc_meta__/owner/2231960876f30c12 898d0c68-f9f7-4d3e-9933-6a32070d1d1c /tidb/cdc/default/__cdc_meta__/owner/2231960876f30ca6 066c2fc4-2313-4926-9ee4-ee0a8ecfed16 /tidb/cdc/default/default/changefeed/info/changefeed-test1 {"upstream-id":7489988041711609721,"namespace":"default","changefeed-id":"changefeed-test1","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-06T08:19:18.176200435+08:00","start-ts":457152596038909964,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457152596051755019} /tidb/cdc/default/default/changefeed/info/changefeed-test2 {"upstream-id":7489988041711609721,"namespace":"default","changefeed-id":"changefeed-test2","sink-uri":"blackhole://","create-time":"2025-04-06T08:19:18.292255943+08:00","start-ts":457152596078231562,"target-ts":0,"admin-job-type":1,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":32},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"stopped","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457152598083371011} /tidb/cdc/default/default/changefeed/info/changefeed-test3 {"upstream-id":7489988041711609721,"namespace":"default","changefeed-id":"changefeed-test3","sink-uri":"blackhole://","create-time":"2025-04-06T08:19:18.475899081+08:00","start-ts":457152596104183818,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457152596130398218} /tidb/cdc/default/default/changefeed/status/changefeed-test1 {"checkpoint-ts":457152596602257438,"min-table-barrier-ts":457152599839735813,"admin-job-type":0} /tidb/cdc/default/default/changefeed/status/changefeed-test2 {"checkpoint-ts":457152596602257438,"min-table-barrier-ts":457152596602257438,"admin-job-type":1} /tidb/cdc/default/default/changefeed/status/changefeed-test3 {"checkpoint-ts":457152596602257438,"min-table-barrier-ts":457152599839735813,"admin-job-type":0} /tidb/cdc/default/default/task/position/066c2fc4-2313-4926-9ee4-ee0a8ecfed16/changefeed-test1 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/066c2fc4-2313-4926-9ee4-ee0a8ecfed16/changefeed-test3 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/898d0c68-f9f7-4d3e-9933-6a32070d1d1c/changefeed-test1 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/898d0c68-f9f7-4d3e-9933-6a32070d1d1c/changefeed-test3 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489988041711609721 {"id":7489988041711609721,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** processors info ***: changefeedID: default/changefeed-test3 {UpstreamID:7489988041711609721 Namespace:default ID:changefeed-test3 SinkURI:blackhole:// CreateTime:2025-04-06 08:19:18.475899081 +0800 CST StartTs:457152596104183818 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003932fc0 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457152596130398218} {CheckpointTs:457152596602257438 MinTableBarrierTs:457152599839735813 AdminJobType:noop} span: {tableID:116, startKey:7480000000000000ff745f720000000000fa, endKey:7480000000000000ff745f730000000000fa}, resolvedTs: 457152596602257438, checkpointTs: 457152596602257438, state: Preparing changefeedID: default/changefeed-test1 {UpstreamID:7489988041711609721 Namespace:default ID:changefeed-test1 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-06 08:19:18.176200435 +0800 CST StartTs:457152596038909964 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc004a2a000 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457152596051755019} {CheckpointTs:457152596602257438 MinTableBarrierTs:457152599839735813 AdminJobType:noop} span: {tableID:112, startKey:7480000000000000ff705f720000000000fa, endKey:7480000000000000ff705f730000000000fa}, resolvedTs: 457152596602257438, checkpointTs: 457152596602257438, state: Preparing span: {tableID:116, startKey:7480000000000000ff745f720000000000fa, endKey:7480000000000000ff745f730000000000fa}, resolvedTs: 457152596602257438, checkpointTs: 457152596602257438, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/066c2fc4-2313-4926-9ee4-ee0a8ecfed16 {"id":"066c2fc4-2313-4926-9ee4-ee0a8ecfed16","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898770} /tidb/cdc/default/__cdc_meta__/capture/898d0c68-f9f7-4d3e-9933-6a32070d1d1c {"id":"898d0c68-f9f7-4d3e-9933-6a32070d1d1c","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898752} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 1 /tidb/cdc/default/__cdc_meta__/owner/2231960876f30c12 898d0c68-f9f7-4d3e-9933-6a32070d1d1c /tidb/cdc/default/__cdc_meta__/owner/2231960876f30ca6 066c2fc4-2313-4926-9ee4-ee0a8ecfed16 /tidb/cdc/default/default/changefeed/info/changefeed-test1 {"upstream-id":7489988041711609721,"namespace":"default","changefeed-id":"changefeed-test1","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-06T08:19:18.176200435+08:00","start-ts":457152596038909964,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457152596051755019} /tidb/cdc/default/default/changefeed/info/changefeed-test2 {"upstream-id":7489988041711609721,"namespace":"default","changefeed-id":"changefeed-test2","sink-uri":"blackhole://","create-time":"2025-04-06T08:19:18.292255943+08:00","start-ts":457152596078231562,"target-ts":0,"admin-job-type":1,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":32},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"stopped","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457152598083371011} /tidb/cdc/default/default/changefeed/info/changefeed-test3 {"upstream-id":7489988041711609721,"namespace":"default","changefeed-id":"changefeed-test3","sink-uri":"blackhole://","create-time":"2025-04-06T08:19:18.475899081+08:00","start-ts":457152596104183818,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457152596130398218} /tidb/cdc/default/default/changefeed/status/changefeed-test1 {"checkpoint-ts":457152596602257438,"min-table-barrier-ts":457152599839735813,"admin-job-type":0} /tidb/cdc/default/default/changefeed/status/changefeed-test2 {"checkpoint-ts":457152596602257438,"min-table-barrier-ts":457152596602257438,"admin-job-type":1} /tidb/cdc/default/default/changefeed/status/changefeed-test3 {"checkpoint-ts":457152596602257438,"min-table-barrier-ts":457152599839735813,"admin-job-type":0} /tidb/cdc/default/default/task/position/066c2fc4-2313-4926-9ee4-ee0a8ecfed16/changefeed-test1 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/066c2fc4-2313-4926-9ee4-ee0a8ecfed16/changefeed-test3 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/898d0c68-f9f7-4d3e-9933-6a32070d1d1c/changefeed-test1 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/898d0c68-f9f7-4d3e-9933-6a32070d1d1c/changefeed-test3 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489988041711609721 {"id":7489988041711609721,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list 2>&1 | grep '"address": "127.0.0.1:8300"' Verifying downstream PD is started... wait process cdc.test exit for 2-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Sun Apr 6 08:19:34 CST 2025] <<<<<< run test case foreign_key success! >>>>>> Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Sun Apr 6 08:19:34 CST 2025] <<<<<< run test case kv_client_stream_reconnect success! >>>>>> "address": "127.0.0.1:8300", run task successfully pass test: list changefeed =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/ddl_manager/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... pass test: get changefeed pass test: pause changefeed VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489988179848361504 TiDB Cluster ID. tikv_gc_leader_uuid 65821de2a680006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-9-92vm1, pid:6619, start at 2025-04-06 08:19:34.178477187 +0800 CST m=+2.460016858 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:21:34.187 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:19:34.170 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:09:34.170 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489988179848361504 TiDB Cluster ID. tikv_gc_leader_uuid 65821de2a680006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-9-92vm1, pid:6619, start at 2025-04-06 08:19:34.178477187 +0800 CST m=+2.460016858 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:21:34.187 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:19:34.170 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:09:34.170 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489988174640766043 TiDB Cluster ID. tikv_gc_leader_uuid 65821de28b8000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-9-92vm1, pid:6687, start at 2025-04-06 08:19:34.079057621 +0800 CST m=+2.301914814 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:21:34.088 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:19:34.130 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:09:34.130 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/capture_suicide_while_balance_table/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/capture_suicide_while_balance_table/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/capture_suicide_while_balance_table/tiflash-proxy.toml"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/capture_suicide_while_balance_table/tiflash/db/proxy"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/capture_suicide_while_balance_table/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Starting Upstream TiDB... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/processor_stop_delay/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) pass test: resume changefeed pass test: rebalance table pass test: move table [Sun Apr 6 08:19:38 CST 2025] <<<<<< START cdc server in capture_suicide_while_balance_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.capture_suicide_while_balance_table.79317933.out server --log-file /tmp/tidb_cdc_test/capture_suicide_while_balance_table/cdc1.log --log-level debug --data-dir /tmp/tidb_cdc_test/capture_suicide_while_balance_table/cdc_data1 --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table sink_retry.finish_mark_1 exists check diff successfully ***************** properties ***************** "mysql.user"="root" "mysql.db"="sink_retry" "scanproportion"="0" "threadcount"="2" "operationcount"="0" "insertproportion"="0" "mysql.port"="4000" "readproportion"="0" "updateproportion"="0" "recordcount"="10" "dotransactions"="false" "mysql.host"="127.0.0.1" "readallfields"="true" "requestdistribution"="uniform" "workload"="core" ********************************************** Run finished, takes 5.045127ms INSERT - Takes(s): 0.0, Count: 10, OPS: 2767.1, Avg(us): 837, Min(us): 509, Max(us): 1872, 95th(us): 2000, 99th(us): 2000 start tidb cluster in /tmp/tidb_cdc_test/ddl_manager Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/processor_stop_delay Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... pass test: get processors Verifying downstream PD is started... pass test: list processors pass test: set log level Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 06 Apr 2025 00:19:41 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/43cf931f-56c8-4419-a884-9799bc396990 {"id":"43cf931f-56c8-4419-a884-9799bc396990","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898778} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319608776d13d6 43cf931f-56c8-4419-a884-9799bc396990 /tidb/cdc/default/default/upstream/7489988179848361504 {"id":7489988179848361504,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/43cf931f-56c8-4419-a884-9799bc396990 {"id":"43cf931f-56c8-4419-a884-9799bc396990","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898778} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319608776d13d6 43cf931f-56c8-4419-a884-9799bc396990 /tidb/cdc/default/default/upstream/7489988179848361504 {"id":7489988179848361504,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/43cf931f-56c8-4419-a884-9799bc396990 {"id":"43cf931f-56c8-4419-a884-9799bc396990","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898778} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319608776d13d6 43cf931f-56c8-4419-a884-9799bc396990 /tidb/cdc/default/default/upstream/7489988179848361504 {"id":7489988179848361504,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Sun Apr 6 08:19:41 CST 2025] <<<<<< START cdc server in capture_suicide_while_balance_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sink/dmlsink/txn/mysql/MySQLSinkHangLongTime=1*return(true)' + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.capture_suicide_while_balance_table.80008002.out server --log-file /tmp/tidb_cdc_test/capture_suicide_while_balance_table/cdc2.log --log-level debug --data-dir /tmp/tidb_cdc_test/capture_suicide_while_balance_table/cdc_data2 --cluster-id default --addr 127.0.0.1:8301 --pd http://127.0.0.1:2379 + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 pass test: remove changefeed pass test: resign owner VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489988194915439133 TiDB Cluster ID. tikv_gc_leader_uuid 65821de7c100009 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-d-7dnbd, pid:4027, start at 2025-04-06 08:19:39.411117141 +0800 CST m=+2.480690217 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:21:39.419 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:19:39.457 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:09:39.457 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489988194915439133 TiDB Cluster ID. tikv_gc_leader_uuid 65821de7c100009 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-d-7dnbd, pid:4027, start at 2025-04-06 08:19:39.411117141 +0800 CST m=+2.480690217 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:21:39.419 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:19:39.457 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:09:39.457 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489988197706939472 TiDB Cluster ID. tikv_gc_leader_uuid 65821de7d040007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-d-7dnbd, pid:4108, start at 2025-04-06 08:19:39.467326153 +0800 CST m=+2.481103319 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:21:39.475 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:19:39.457 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:09:39.457 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/changefeed_reconstruct/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/changefeed_reconstruct/tiflash/log/error.log arg matches is ArgMatches { args: {"pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/changefeed_reconstruct/tiflash-proxy.toml"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/changefeed_reconstruct/tiflash/db/proxy"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/changefeed_reconstruct/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Verifying downstream PD is started... Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Cache restored successfully (ws/jenkins-pingcap-tiflow-periodics_cdc_integration_mysql_test-234/tiflow-cdc) 4028702720 bytes in 11.34 secs (355275594 bytes/sec) [Pipeline] { [Pipeline] cache [Sun Apr 6 08:19:43 CST 2025] <<<<<< START cdc server in changefeed_reconstruct case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_reconstruct.55025504.out server --log-file /tmp/tidb_cdc_test/changefeed_reconstruct/cdcserver1.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_reconstruct/cdc_dataserver1 --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 06 Apr 2025 00:19:44 GMT < Content-Length: 1281 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/43cf931f-56c8-4419-a884-9799bc396990 {"id":"43cf931f-56c8-4419-a884-9799bc396990","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898778} /tidb/cdc/default/__cdc_meta__/capture/faccd33d-46d6-46e2-b5ff-5a223650a631 {"id":"faccd33d-46d6-46e2-b5ff-5a223650a631","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898781} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319608776d13d6 43cf931f-56c8-4419-a884-9799bc396990 /tidb/cdc/default/__cdc_meta__/owner/22319608776d1415 faccd33d-46d6-46e2-b5ff-5a223650a631 /tidb/cdc/default/default/upstream/7489988179848361504 {"id":7489988179848361504,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/43cf931f-56c8-4419-a884-9799bc396990 {"id":"43cf931f-56c8-4419-a884-9799bc396990","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898778} /tidb/cdc/default/__cdc_meta__/capture/faccd33d-46d6-46e2-b5ff-5a223650a631 {"id":"faccd33d-46d6-46e2-b5ff-5a223650a631","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898781} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319608776d13d6 43cf931f-56c8-4419-a884-9799bc396990 /tidb/cdc/default/__cdc_meta__/owner/22319608776d1415 faccd33d-46d6-46e2-b5ff-5a223650a631 /tidb/cdc/default/default/upstream/7489988179848361504 {"id":7489988179848361504,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/43cf931f-56c8-4419-a884-9799bc396990 {"id":"43cf931f-56c8-4419-a884-9799bc396990","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898778} /tidb/cdc/default/__cdc_meta__/capture/faccd33d-46d6-46e2-b5ff-5a223650a631 {"id":"faccd33d-46d6-46e2-b5ff-5a223650a631","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898781} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319608776d13d6 43cf931f-56c8-4419-a884-9799bc396990 /tidb/cdc/default/__cdc_meta__/owner/22319608776d1415 faccd33d-46d6-46e2-b5ff-5a223650a631 /tidb/cdc/default/default/upstream/7489988179848361504 {"id":7489988179848361504,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x table capture_suicide_while_balance_table.t1 not exists for 1-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/ddl_puller_lag/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/cdc/run.sh using Sink-Type: mysql... <<================= ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 06 Apr 2025 00:19:46 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ae244744-aca3-45b3-a02c-e1c3f65fa506 {"id":"ae244744-aca3-45b3-a02c-e1c3f65fa506","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898783} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319608777f5bd8 ae244744-aca3-45b3-a02c-e1c3f65fa506 /tidb/cdc/default/default/upstream/7489988194915439133 {"id":7489988194915439133,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ae244744-aca3-45b3-a02c-e1c3f65fa506 {"id":"ae244744-aca3-45b3-a02c-e1c3f65fa506","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898783} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319608777f5bd8 ae244744-aca3-45b3-a02c-e1c3f65fa506 /tidb/cdc/default/default/upstream/7489988194915439133 {"id":7489988194915439133,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ae244744-aca3-45b3-a02c-e1c3f65fa506 {"id":"ae244744-aca3-45b3-a02c-e1c3f65fa506","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898783} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319608777f5bd8 ae244744-aca3-45b3-a02c-e1c3f65fa506 /tidb/cdc/default/default/upstream/7489988194915439133 {"id":7489988194915439133,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x table capture_suicide_while_balance_table.t1 exists table capture_suicide_while_balance_table.t2 not exists for 1-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) The 1 times to try to start tidb cluster... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489988224941036897 TiDB Cluster ID. tikv_gc_leader_uuid 65821dedd640014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-v-hm9ds, pid:3780, start at 2025-04-06 08:19:45.674149936 +0800 CST m=+2.665121816 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:21:45.683 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:19:45.725 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:09:45.725 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489988224941036897 TiDB Cluster ID. tikv_gc_leader_uuid 65821dedd640014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-v-hm9ds, pid:3780, start at 2025-04-06 08:19:45.674149936 +0800 CST m=+2.665121816 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:21:45.683 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:19:45.725 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:09:45.725 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489988228744850538 TiDB Cluster ID. tikv_gc_leader_uuid 65821dee06c000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-v-hm9ds, pid:3844, start at 2025-04-06 08:19:45.860581356 +0800 CST m=+2.789423239 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:21:45.869 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:19:45.869 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:09:45.869 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/ddl_manager/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/ddl_manager/tiflash/log/error.log arg matches is ArgMatches { args: {"addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/ddl_manager/tiflash-proxy.toml"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/ddl_manager/tiflash/db/proxy"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/ddl_manager/tiflash/log/proxy.log"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } start tidb cluster in /tmp/tidb_cdc_test/ddl_puller_lag Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... ***************** properties ***************** "dotransactions"="false" "operationcount"="0" "workload"="core" "recordcount"="50" "mysql.user"="root" "mysql.db"="changefeed_reconstruct" "readproportion"="0" "threadcount"="4" "mysql.port"="4000" "scanproportion"="0" "insertproportion"="0" "updateproportion"="0" "mysql.host"="127.0.0.1" "readallfields"="true" "requestdistribution"="uniform" ********************************************** Run finished, takes 22.444388ms INSERT - Takes(s): 0.0, Count: 48, OPS: 2920.0, Avg(us): 1764, Min(us): 1136, Max(us): 5877, 95th(us): 6000, 99th(us): 6000 table changefeed_reconstruct.usertable not exists for 1-th check, retry later table capture_suicide_while_balance_table.t2 exists table capture_suicide_while_balance_table.t3 not exists for 1-th check, retry later Verifying downstream PD is started... [Sun Apr 6 08:19:49 CST 2025] <<<<<< START cdc server in ddl_manager case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ExecuteDDLSlowly=return(true)' + '[' -z '' ']' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_manager.52155217.out server --log-file /tmp/tidb_cdc_test/ddl_manager/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_manager/cdc_data --cluster-id default + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489988228927057726 TiDB Cluster ID. tikv_gc_leader_uuid 65821defd50000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-k-bx9s4, pid:6473, start at 2025-04-06 08:19:47.705639934 +0800 CST m=+2.591082698 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:21:47.714 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:19:47.718 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:09:47.718 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489988228927057726 TiDB Cluster ID. tikv_gc_leader_uuid 65821defd50000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-k-bx9s4, pid:6473, start at 2025-04-06 08:19:47.705639934 +0800 CST m=+2.591082698 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:21:47.714 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:19:47.718 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:09:47.718 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489988237517503777 TiDB Cluster ID. tikv_gc_leader_uuid 65821defec0000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-k-bx9s4, pid:6546, start at 2025-04-06 08:19:47.787434344 +0800 CST m=+2.612762671 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:21:47.795 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:19:47.760 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:09:47.760 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/processor_stop_delay/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/processor_stop_delay/tiflash/log/error.log arg matches is ArgMatches { args: {"addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/processor_stop_delay/tiflash/log/proxy.log"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/processor_stop_delay/tiflash-proxy.toml"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/processor_stop_delay/tiflash/db/proxy"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table changefeed_reconstruct.usertable exists check diff failed 1-th time, retry later Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table capture_suicide_while_balance_table.t3 exists table capture_suicide_while_balance_table.t4 not exists for 1-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/cdc Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... [Sun Apr 6 08:19:51 CST 2025] <<<<<< START cdc server in processor_stop_delay case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/processor/processorStopDelay=1*sleep(10000)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.processor_stop_delay.79357937.out server --log-file /tmp/tidb_cdc_test/processor_stop_delay/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/processor_stop_delay/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... check diff successfully wait process 5507 exit for 1-th time... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 06 Apr 2025 00:19:52 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3f945389-c049-4648-8ffa-861f355685bd {"id":"3f945389-c049-4648-8ffa-861f355685bd","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898789} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319608779cf4d8 3f945389-c049-4648-8ffa-861f355685bd /tidb/cdc/default/default/upstream/7489988224941036897 {"id":7489988224941036897,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3f945389-c049-4648-8ffa-861f355685bd {"id":"3f945389-c049-4648-8ffa-861f355685bd","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898789} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319608779cf4d8 3f945389-c049-4648-8ffa-861f355685bd /tidb/cdc/default/default/upstream/7489988224941036897 {"id":7489988224941036897,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3f945389-c049-4648-8ffa-861f355685bd {"id":"3f945389-c049-4648-8ffa-861f355685bd","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898789} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319608779cf4d8 3f945389-c049-4648-8ffa-861f355685bd /tidb/cdc/default/default/upstream/7489988224941036897 {"id":7489988224941036897,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_manager.cli.5272.out cli changefeed create --sink-uri=mysql://root@127.0.0.1:3306/ -c=ddl-manager table capture_suicide_while_balance_table.t4 exists wait process 5507 exit for 2-th time... Create changefeed successfully! ID: ddl-manager Info: {"upstream_id":7489988224941036897,"namespace":"default","id":"ddl-manager","sink_uri":"mysql://root@127.0.0.1:3306/","create_time":"2025-04-06T08:19:53.003671318+08:00","start_ts":457152605159424004,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457152605159424004,"checkpoint_ts":457152605159424004,"checkpoint_time":"2025-04-06 08:19:52.875"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (5507) - No such process wait process 5507 exit for 3-th time... process 5507 already exit check_no_capture http://127.0.0.1:2379 parse error: Invalid numeric literal at line 1, column 6 run task successfully [Sun Apr 6 08:19:53 CST 2025] <<<<<< START cdc server in changefeed_reconstruct case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_reconstruct.57545756.out server --log-file /tmp/tidb_cdc_test/changefeed_reconstruct/cdcserver2.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_reconstruct/cdc_dataserver2 --cluster-id default --addr 127.0.0.1:8300 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Cache restored successfully (ws/jenkins-pingcap-tiflow-periodics_cdc_integration_mysql_test-234/tiflow-cdc) 4028702720 bytes in 8.99 secs (448068946 bytes/sec) [Pipeline] { [Pipeline] cache Starting Upstream TiDB... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 06 Apr 2025 00:19:54 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6f057386-18bd-42e1-ac82-cfa835667106 {"id":"6f057386-18bd-42e1-ac82-cfa835667106","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898791} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/2231960877a139d8 6f057386-18bd-42e1-ac82-cfa835667106 /tidb/cdc/default/default/upstream/7489988228927057726 {"id":7489988228927057726,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6f057386-18bd-42e1-ac82-cfa835667106 {"id":"6f057386-18bd-42e1-ac82-cfa835667106","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898791} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/2231960877a139d8 6f057386-18bd-42e1-ac82-cfa835667106 /tidb/cdc/default/default/upstream/7489988228927057726 {"id":7489988228927057726,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6f057386-18bd-42e1-ac82-cfa835667106 {"id":"6f057386-18bd-42e1-ac82-cfa835667106","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898791} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/2231960877a139d8 6f057386-18bd-42e1-ac82-cfa835667106 /tidb/cdc/default/default/upstream/7489988228927057726 {"id":7489988228927057726,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 140 100 35 100 105 725 2175 --:--:-- --:--:-- --:--:-- 2187 { "status": true, "message": "" table processor_stop_delay.t not exists for 1-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 06 Apr 2025 00:19:56 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/96a1c920-2498-4d09-8547-79dd3a561965 {UpstreamID:7489988194915439133 Namespace:default ID:96a1c920-2498-4d09-8547-79dd3a561965 SinkURI:mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1 CreateTime:2025-04-06 08:19:46.569311604 +0800 CST StartTs:457152603474362376 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003f519e0 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457152603500314629} {CheckpointTs:457152604680224772 MinTableBarrierTs:457152605990682628 AdminJobType:noop} span: {tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}, resolvedTs: 457152604680224772, checkpointTs: 457152604680224772, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f4942f7d-2400-4623-b8ff-4a0bda6325aa {"id":"f4942f7d-2400-4623-b8ff-4a0bda6325aa","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898793} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319608777f5c81 f4942f7d-2400-4623-b8ff-4a0bda6325aa /tidb/cdc/default/default/changefeed/info/96a1c920-2498-4d09-8547-79dd3a561965 {"upstream-id":7489988194915439133,"namespace":"default","changefeed-id":"96a1c920-2498-4d09-8547-79dd3a561965","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2025-04-06T08:19:46.569311604+08:00","start-ts":457152603474362376,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457152603500314629} /tidb/cdc/default/default/changefeed/status/96a1c920-2498-4d09-8547-79dd3a561965 {"checkpoint-ts":457152604680224772,"min-table-barrier-ts":457152605990682628,"admin-job-type":0} /tidb/cdc/default/default/task/position/f4942f7d-2400-4623-b8ff-4a0bda6325aa/96a1c920-2498-4d09-8547-79dd3a561965 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489988194915439133 {"id":7489988194915439133,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/96a1c920-2498-4d09-8547-79dd3a561965 {UpstreamID:7489988194915439133 Namespace:default ID:96a1c920-2498-4d09-8547-79dd3a561965 SinkURI:mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1 CreateTime:2025-04-06 08:19:46.569311604 +0800 CST StartTs:457152603474362376 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003f519e0 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457152603500314629} {CheckpointTs:457152604680224772 MinTableBarrierTs:457152605990682628 AdminJobType:noop} span: {tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}, resolvedTs: 457152604680224772, checkpointTs: 457152604680224772, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f4942f7d-2400-4623-b8ff-4a0bda6325aa {"id":"f4942f7d-2400-4623-b8ff-4a0bda6325aa","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898793} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319608777f5c81 f4942f7d-2400-4623-b8ff-4a0bda6325aa /tidb/cdc/default/default/changefeed/info/96a1c920-2498-4d09-8547-79dd3a561965 {"upstream-id":7489988194915439133,"namespace":"default","changefeed-id":"96a1c920-2498-4d09-8547-79dd3a561965","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2025-04-06T08:19:46.569311604+08:00","start-ts":457152603474362376,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457152603500314629} /tidb/cdc/default/default/changefeed/status/96a1c920-2498-4d09-8547-79dd3a561965 {"checkpoint-ts":457152604680224772,"min-table-barrier-ts":457152605990682628,"admin-job-type":0} /tidb/cdc/default/default/task/position/f4942f7d-2400-4623-b8ff-4a0bda6325aa/96a1c920-2498-4d09-8547-79dd3a561965 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489988194915439133 + grep -q 'failed to get info:' {"id":7489988194915439133,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/96a1c920-2498-4d09-8547-79dd3a561965 {UpstreamID:7489988194915439133 Namespace:default ID:96a1c920-2498-4d09-8547-79dd3a561965 SinkURI:mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1 CreateTime:2025-04-06 08:19:46.569311604 +0800 CST StartTs:457152603474362376 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003f519e0 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457152603500314629} {CheckpointTs:457152604680224772 MinTableBarrierTs:457152605990682628 AdminJobType:noop} span: {tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}, resolvedTs: 457152604680224772, checkpointTs: 457152604680224772, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f4942f7d-2400-4623-b8ff-4a0bda6325aa {"id":"f4942f7d-2400-4623-b8ff-4a0bda6325aa","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898793} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319608777f5c81 f4942f7d-2400-4623-b8ff-4a0bda6325aa /tidb/cdc/default/default/changefeed/info/96a1c920-2498-4d09-8547-79dd3a561965 {"upstream-id":7489988194915439133,"namespace":"default","changefeed-id":"96a1c920-2498-4d09-8547-79dd3a561965","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2025-04-06T08:19:46.569311604+08:00","start-ts":457152603474362376,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457152603500314629} /tidb/cdc/default/default/changefeed/status/96a1c920-2498-4d09-8547-79dd3a561965 {"checkpoint-ts":457152604680224772,"min-table-barrier-ts":457152605990682628,"admin-job-type":0} /tidb/cdc/default/default/task/position/f4942f7d-2400-4623-b8ff-4a0bda6325aa/96a1c920-2498-4d09-8547-79dd3a561965 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489988194915439133 + grep -q 'etcd info' {"id":7489988194915439133,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list --pd=http://127.0.0.1:2379 2>&1 | grep id VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489988263644318968 TiDB Cluster ID. tikv_gc_leader_uuid 65821df68840001 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-c-m09xf, pid:3988, start at 2025-04-06 08:19:54.530363218 +0800 CST m=+2.163704758 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:21:54.539 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:19:54.529 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:09:54.529 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489988263644318968 TiDB Cluster ID. tikv_gc_leader_uuid 65821df68840001 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-c-m09xf, pid:3988, start at 2025-04-06 08:19:54.530363218 +0800 CST m=+2.163704758 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:21:54.539 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:19:54.529 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:09:54.529 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489988270255655756 TiDB Cluster ID. tikv_gc_leader_uuid 65821df6a14000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-c-m09xf, pid:4069, start at 2025-04-06 08:19:54.671094646 +0800 CST m=+2.224302550 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:21:54.680 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:19:54.672 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:09:54.672 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/ddl_puller_lag/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/ddl_puller_lag/tiflash/log/error.log arg matches is ArgMatches { args: {"addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/ddl_puller_lag/tiflash/log/proxy.log"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/ddl_puller_lag/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/ddl_puller_lag/tiflash-proxy.toml"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } }lease 22319608776d1415 revoked check diff failed 1-th time, retry later table processor_stop_delay.t exists check diff failed 1-th time, retry later "id": "f4942f7d-2400-4623-b8ff-4a0bda6325aa", "cluster-id": "default" run task successfully + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_puller_lag.cli.5310.out cli tso query --pd=http://127.0.0.1:2379 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489988277111646461 TiDB Cluster ID. tikv_gc_leader_uuid 65821df88780009 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-v-nsng0, pid:4189, start at 2025-04-06 08:19:56.58752471 +0800 CST m=+2.171429351 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:21:56.634 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:19:56.627 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:09:56.627 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489988277111646461 TiDB Cluster ID. tikv_gc_leader_uuid 65821df88780009 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-v-nsng0, pid:4189, start at 2025-04-06 08:19:56.58752471 +0800 CST m=+2.171429351 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:21:56.634 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:19:56.627 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:09:56.627 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489988279084127680 TiDB Cluster ID. tikv_gc_leader_uuid 65821df8a100005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-v-nsng0, pid:4269, start at 2025-04-06 08:19:56.683082936 +0800 CST m=+2.212373720 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:21:56.728 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:19:56.727 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:09:56.727 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/cdc/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/cdc/tiflash/log/error.log arg matches is ArgMatches { args: {"memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/cdc/tiflash/log/proxy.log"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/cdc/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/cdc/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } check diff failed 2-th time, retry later capture_id: f4942f7d-2400-4623-b8ff-4a0bda6325aa check_processor_table_count http://127.0.0.1:2379 96a1c920-2498-4d09-8547-79dd3a561965 f4942f7d-2400-4623-b8ff-4a0bda6325aa 1 run task successfully check diff successfully check_processor_table_count http://127.0.0.1:2379 96a1c920-2498-4d09-8547-79dd3a561965 f4942f7d-2400-4623-b8ff-4a0bda6325aa 0 table count 1 does equal to expected count 0 run task failed 1-th time, retry later + set +x + tso='457152606785241089 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 457152606785241089 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Sun Apr 6 08:20:00 CST 2025] <<<<<< START cdc server in ddl_puller_lag case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/processor/processorDDLResolved=1*sleep(180000)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_puller_lag.53555357.out server --log-file /tmp/tidb_cdc_test/ddl_puller_lag/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_puller_lag/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [Sun Apr 6 08:20:00 CST 2025] <<<<<< START cdc server in cdc case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cdc.55125514.out server --log-file /tmp/tidb_cdc_test/cdc/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/cdc/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff failed 3-th time, retry later check diff failed 1-th time, retry later check_processor_table_count http://127.0.0.1:2379 96a1c920-2498-4d09-8547-79dd3a561965 f4942f7d-2400-4623-b8ff-4a0bda6325aa 0 run task successfully check diff failed 2-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 06 Apr 2025 00:20:03 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/38bbcb52-116f-47f9-9c61-5d14a58940e6 {"id":"38bbcb52-116f-47f9-9c61-5d14a58940e6","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898800} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/2231960877bf87f9 38bbcb52-116f-47f9-9c61-5d14a58940e6 /tidb/cdc/default/default/upstream/7489988263644318968 {"id":7489988263644318968,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/38bbcb52-116f-47f9-9c61-5d14a58940e6 {"id":"38bbcb52-116f-47f9-9c61-5d14a58940e6","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898800} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/2231960877bf87f9 38bbcb52-116f-47f9-9c61-5d14a58940e6 /tidb/cdc/default/default/upstream/7489988263644318968 {"id":7489988263644318968,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/38bbcb52-116f-47f9-9c61-5d14a58940e6 {"id":"38bbcb52-116f-47f9-9c61-5d14a58940e6","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898800} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/2231960877bf87f9 38bbcb52-116f-47f9-9c61-5d14a58940e6 /tidb/cdc/default/default/upstream/7489988263644318968 {"id":7489988263644318968,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_puller_lag.cli.5414.out cli changefeed create --start-ts=457152606785241089 --sink-uri=mysql+ssl://normal:123456@127.0.0.1:3306/ Create changefeed successfully! ID: 7c294623-bff3-4d0e-bee4-eb7ee5a9c782 Info: {"upstream_id":7489988263644318968,"namespace":"default","id":"7c294623-bff3-4d0e-bee4-eb7ee5a9c782","sink_uri":"mysql+ssl://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-06T08:20:04.020295207+08:00","start_ts":457152606785241089,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457152606785241089,"checkpoint_ts":457152606785241089,"checkpoint_time":"2025-04-06 08:19:59.077"} PASS ***************** properties ***************** "requestdistribution"="uniform" "threadcount"="4" "mysql.db"="changefeed_reconstruct" "mysql.host"="127.0.0.1" "mysql.user"="root" "workload"="core" "scanproportion"="0" "mysql.port"="4000" "readallfields"="true" "readproportion"="0" "recordcount"="50" "operationcount"="0" "updateproportion"="0" "insertproportion"="0" "dotransactions"="false" ********************************************** Run finished, takes 20.097187ms INSERT - Takes(s): 0.0, Count: 48, OPS: 3106.6, Avg(us): 1607, Min(us): 1106, Max(us): 4721, 95th(us): 5000, 99th(us): 5000 table changefeed_reconstruct.usertable not exists for 1-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 06 Apr 2025 00:20:03 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a70c74c2-a79c-4246-a7a5-2707435bb351 {"id":"a70c74c2-a79c-4246-a7a5-2707435bb351","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898801} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/2231960877cba2d6 a70c74c2-a79c-4246-a7a5-2707435bb351 /tidb/cdc/default/default/upstream/7489988277111646461 {"id":7489988277111646461,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a70c74c2-a79c-4246-a7a5-2707435bb351 {"id":"a70c74c2-a79c-4246-a7a5-2707435bb351","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898801} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/2231960877cba2d6 a70c74c2-a79c-4246-a7a5-2707435bb351 /tidb/cdc/default/default/upstream/7489988277111646461 {"id":7489988277111646461,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a70c74c2-a79c-4246-a7a5-2707435bb351 {"id":"a70c74c2-a79c-4246-a7a5-2707435bb351","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898801} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/2231960877cba2d6 a70c74c2-a79c-4246-a7a5-2707435bb351 /tidb/cdc/default/default/upstream/7489988277111646461 {"id":7489988277111646461,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cdc.cli.5577.out cli changefeed create --sink-uri=mysql://normal:123456@127.0.0.1:3306/ --config /tmp/tidb_cdc_test/cdc/pulsar_test.toml coverage: 2.4% of statements in github.com/pingcap/tiflow/... check diff successfully wait process cdc.test exit for 1-th time... Create changefeed successfully! ID: 0b9ee1e7-2e47-4c09-99be-d8ee0f7d474c Info: {"upstream_id":7489988277111646461,"namespace":"default","id":"0b9ee1e7-2e47-4c09-99be-d8ee0f7d474c","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-06T08:20:04.248994291+08:00","start_ts":457152608121651203,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457152608121651203,"checkpoint_ts":457152608121651203,"checkpoint_time":"2025-04-06 08:20:04.175"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... process cdc.test already exit [Sun Apr 6 08:20:05 CST 2025] <<<<<< run test case capture_suicide_while_balance_table success! >>>>>> + set +x + set +x go: downloading go1.23.4 (linux/amd64) table changefeed_reconstruct.usertable exists check diff failed 1-th time, retry later check diff failed 3-th time, retry later check diff failed 4-th time, retry later Cache restored successfully (ws/jenkins-pingcap-tiflow-periodics_cdc_integration_mysql_test-234/tiflow-cdc) 4028702720 bytes in 12.31 secs (327402457 bytes/sec) [Pipeline] { [Pipeline] cache check diff successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Sun Apr 6 08:20:09 CST 2025] <<<<<< run test case changefeed_reconstruct success! >>>>>> check diff failed 5-th time, retry later table sink_retry.finish_mark_2 exists check diff successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Sun Apr 6 08:20:07 CST 2025] <<<<<< run test case sink_retry success! >>>>>> go: downloading github.com/go-sql-driver/mysql v1.7.1 go: downloading github.com/pingcap/errors v0.11.5-0.20240318064555-6bd07397691f go: downloading github.com/pingcap/log v1.1.1-0.20241212030209-7e3ff8601a2a go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20241223052309-3735ed55a394 go: downloading github.com/pingcap/tidb/pkg/parser v0.0.0-20241223052309-3735ed55a394 go: downloading github.com/BurntSushi/toml v1.4.1-0.20240526193622-a339e1f7089c go: downloading go.uber.org/zap v1.27.0 go: downloading github.com/pingcap/failpoint v0.0.0-20240528011301-b51a646c7c86 go: downloading gopkg.in/natefinch/lumberjack.v2 v2.2.1 go: downloading go.uber.org/atomic v1.11.0 go: downloading go.uber.org/multierr v1.11.0 wait process 5220 exit for 1-th time... wait process 5220 exit for 2-th time... wait process 5220 exit for 3-th time... wait process 5220 exit for 4-th time... wait process 5220 exit for 5-th time... wait process 5220 exit for 6-th time... check diff failed 6-th time, retry later go: downloading golang.org/x/text v0.21.0 go: downloading github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 go: downloading github.com/json-iterator/go v1.1.12 go: downloading github.com/pingcap/kvproto v0.0.0-20241120022153-92b0414aeed8 go: downloading github.com/tikv/pd/client v0.0.0-20241111073742-238d4d79ea31 go: downloading github.com/gorilla/mux v1.8.1 go: downloading github.com/spf13/pflag v1.0.5 go: downloading gopkg.in/yaml.v2 v2.4.0 go: downloading github.com/coocood/freecache v1.2.1 go: downloading github.com/tikv/client-go/v2 v2.0.8-0.20241209094930-06d7f4b9233b go: downloading go.etcd.io/etcd/client/v3 v3.5.12 go: downloading github.com/prometheus/client_golang v1.20.5 go: downloading golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 go: downloading github.com/docker/go-units v0.5.0 go: downloading github.com/golang/protobuf v1.5.4 go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible go: downloading github.com/stretchr/testify v1.10.0 go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 go: downloading github.com/google/btree v1.1.2 go: downloading github.com/pingcap/sysutil v1.0.1-0.20240311050922-ae81ee01f3a5 go: downloading github.com/opentracing/opentracing-go v1.2.0 go: downloading github.com/otiai10/copy v1.2.0 go: downloading github.com/bits-and-blooms/bitset v1.14.3 go: downloading golang.org/x/sys v0.28.0 go: downloading github.com/pingcap/tipb v0.0.0-20241105053214-f91fdb81a69e wait process 5220 exit for 7-th time... go: downloading github.com/coreos/go-semver v0.3.1 go: downloading google.golang.org/grpc v1.64.0 go: downloading github.com/shirou/gopsutil/v3 v3.24.5 go: downloading golang.org/x/net v0.33.0 go: downloading github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a go: downloading github.com/google/uuid v1.6.0 go: downloading golang.org/x/sync v0.10.0 go: downloading github.com/prometheus/client_model v0.6.1 go: downloading github.com/tidwall/btree v1.7.0 go: downloading github.com/danjacques/gofslock v0.0.0-20240212154529-d899e02bfe22 go: downloading github.com/tikv/pd v1.1.0-beta.0.20240407022249-7179657d129b go: downloading github.com/influxdata/tdigest v0.0.1 go: downloading github.com/opentracing/basictracer-go v1.1.0 go: downloading github.com/twmb/murmur3 v1.1.6 go: downloading github.com/yangkeao/ldap/v3 v3.4.5-0.20230421065457-369a3bab1117 go: downloading github.com/gogo/protobuf v1.3.2 go: downloading github.com/qri-io/jsonschema v0.2.1 go: downloading github.com/dolthub/swiss v0.2.1 go: downloading github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef go: downloading github.com/cespare/xxhash/v2 v2.3.0 go: downloading cloud.google.com/go/storage v1.39.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v1.16.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 go: downloading github.com/aws/aws-sdk-go v1.55.5 go: downloading github.com/go-resty/resty/v2 v2.11.0 go: downloading github.com/klauspost/compress v1.17.9 go: downloading github.com/ks3sdklib/aws-sdk-go v1.2.9 go: downloading golang.org/x/oauth2 v0.24.0 go: downloading google.golang.org/api v0.170.0 go: downloading github.com/pkg/errors v0.9.1 go: downloading google.golang.org/protobuf v1.35.2 go: downloading github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 go: downloading cloud.google.com/go v0.112.2 wait process 5220 exit for 8-th time... pass test: get tso wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... go: downloading golang.org/x/tools v0.28.0 cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Sun Apr 6 08:20:13 CST 2025] <<<<<< run test case http_api success! >>>>>> go: downloading go.etcd.io/etcd/api/v3 v3.5.12 go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.12 go: downloading go.uber.org/mock v0.4.0 go: downloading github.com/golang/snappy v0.0.4 go: downloading github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc go: downloading github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 go: downloading github.com/cockroachdb/pebble v1.1.0 go: downloading github.com/jfcg/sorty/v2 v2.1.0 wait process 5220 exit for 9-th time... go: downloading golang.org/x/time v0.7.0 go: downloading github.com/carlmjohnson/flagext v0.21.0 go: downloading github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 go: downloading github.com/go-asn1-ber/asn1-ber v1.5.4 go: downloading github.com/dolthub/maphash v0.1.0 go: downloading github.com/beorn7/perks v1.0.1 go: downloading github.com/prometheus/common v0.61.0 go: downloading github.com/prometheus/procfs v0.15.1 go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible go: downloading github.com/tklauser/go-sysconf v0.3.12 go: downloading github.com/qri-io/jsonpointer v0.1.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 go: downloading github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 go: downloading golang.org/x/crypto v0.31.0 go: downloading github.com/dgraph-io/ristretto v0.1.1 go: downloading github.com/pingcap/badger v1.5.1-0.20241015064302-38533b6cbf8d go: downloading github.com/dgryski/go-farm v0.0.0-20240924180020-3414d57e47da go: downloading cloud.google.com/go/compute/metadata v0.3.0 go: downloading github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df go: downloading github.com/lestrrat-go/jwx/v2 v2.0.21 go: downloading github.com/jellydator/ttlcache/v3 v3.0.1 go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading cloud.google.com/go/iam v1.1.7 go: downloading github.com/googleapis/gax-go/v2 v2.12.3 go: downloading google.golang.org/genproto v0.0.0-20240401170217-c3f982113cda go: downloading google.golang.org/genproto/googleapis/api v0.0.0-20240401170217-c3f982113cda go: downloading github.com/cloudfoundry/gosigar v1.3.6 go: downloading github.com/spkg/bom v1.0.0 go: downloading github.com/xitongsys/parquet-go v1.6.3-0.20240520233950-75e935fc3e17 go: downloading github.com/coreos/go-systemd/v22 v22.5.0 go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading github.com/jfcg/sixb v1.3.8 go: downloading github.com/jedib0t/go-pretty/v6 v6.2.2 go: downloading google.golang.org/genproto/googleapis/rpc v0.0.0-20240515191416-fc5f0ca64291 go: downloading github.com/google/pprof v0.0.0-20241001023024-f4c0cfd0cf1d go: downloading github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f go: downloading github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 go: downloading github.com/tklauser/numcpus v0.6.1 go: downloading github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 wait process 5220 exit for 10-th time... check diff successfully go: downloading github.com/kylelemons/godebug v1.1.0 go: downloading github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c go: downloading github.com/cheggaaa/pb/v3 v3.0.8 go: downloading golang.org/x/term v0.27.0 go: downloading github.com/robfig/cron/v3 v3.0.1 go: downloading go.opencensus.io v0.23.1-0.20220331163232-052120675fac go: downloading go.opentelemetry.io/otel v1.24.0 go: downloading go.opentelemetry.io/otel/trace v1.24.0 go: downloading github.com/robfig/cron v1.2.0 wait process 5220 exit for 11-th time... go: downloading github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec go: downloading github.com/lestrrat-go/blackmagic v1.0.2 go: downloading github.com/lestrrat-go/httprc v1.0.5 go: downloading github.com/lestrrat-go/iter v1.0.2 go: downloading github.com/lestrrat-go/option v1.0.1 go: downloading github.com/apache/thrift v0.16.0 go: downloading github.com/golang-jwt/jwt/v5 v5.2.1 go: downloading github.com/mattn/go-runewidth v0.0.16 go: downloading github.com/dustin/go-humanize v1.0.1 go: downloading github.com/golang/glog v1.2.0 go: downloading github.com/VividCortex/ewma v1.2.0 go: downloading github.com/fatih/color v1.18.0 go: downloading github.com/mattn/go-colorable v0.1.13 go: downloading github.com/mattn/go-isatty v0.0.20 go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da go: downloading github.com/lestrrat-go/httpcc v1.0.1 go: downloading github.com/ncw/directio v1.0.5 go: downloading github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 go: downloading github.com/golang-jwt/jwt v3.2.2+incompatible go: downloading github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 go: downloading github.com/apache/arrow/go/v12 v12.0.1 go: downloading github.com/goccy/go-reflect v1.2.0 go: downloading github.com/go-logr/logr v1.4.1 go: downloading go.opentelemetry.io/otel/metric v1.24.0 go: downloading github.com/rivo/uniseg v0.4.7 go: downloading github.com/klauspost/cpuid v1.3.1 go: downloading github.com/pierrec/lz4/v4 v4.1.18 go: downloading github.com/go-logr/stdr v1.2.2 go: downloading github.com/cockroachdb/errors v1.11.1 go: downloading github.com/DataDog/zstd v1.5.5 go: downloading github.com/cockroachdb/redact v1.1.5 go: downloading github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 wait process cdc.test exit for 1-th time... /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (5220) - No such process wait process 5220 exit for 12-th time... process 5220 already exit [Sun Apr 6 08:20:15 CST 2025] <<<<<< START cdc server in ddl_manager case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ExecuteDDLSlowly=return(true)' + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_manager.54035405.out server --log-file /tmp/tidb_cdc_test/ddl_manager/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_manager/cdc_data --cluster-id default + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 go: downloading github.com/kr/pretty v0.3.1 go: downloading github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b go: downloading github.com/getsentry/sentry-go v0.27.0 go: downloading github.com/rogpeppe/go-internal v1.13.1 go: downloading github.com/kr/text v0.2.0 wait process cdc.test exit for 2-th time... go: downloading golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 go: downloading github.com/google/flatbuffers v2.0.8+incompatible go: downloading github.com/goccy/go-json v0.10.2 go: downloading github.com/zeebo/xxh3 v1.0.2 go: downloading github.com/andybalholm/brotli v1.0.5 cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Sun Apr 6 08:20:15 CST 2025] <<<<<< run test case processor_stop_delay success! >>>>>> go: downloading github.com/klauspost/cpuid/v2 v2.2.4 go: downloading github.com/pierrec/lz4 v2.6.1+incompatible =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/syncpoint/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > Cache restored successfully (ws/jenkins-pingcap-tiflow-periodics_cdc_integration_mysql_test-234/tiflow-cdc) 4028702720 bytes in 8.59 secs (469061162 bytes/sec) [Pipeline] { [Pipeline] cache go: downloading github.com/googleapis/enterprise-certificate-proxy v0.3.2 go: downloading go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 go: downloading github.com/google/s2a-go v0.1.7 go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 go: downloading github.com/felixge/httpsnoop v1.0.4 go: downloading github.com/jmespath/go-jmespath v0.4.0 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/multi_capture/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/changefeed_error/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/syncpoint Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... < HTTP/1.1 200 OK < Date: Sun, 06 Apr 2025 00:20:20 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/ddl-manager {UpstreamID:7489988224941036897 Namespace:default ID:ddl-manager SinkURI:mysql://root@127.0.0.1:3306/ CreateTime:2025-04-06 08:19:53.003671318 +0800 CST StartTs:457152605159424004 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00441a240 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457152605185376262} {CheckpointTs:457152605854105601 MinTableBarrierTs:457152605854105601 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2319dd06-03eb-41fa-a3a3-35d2947b4432 {"id":"2319dd06-03eb-41fa-a3a3-35d2947b4432","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898815} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319608779cf722 2319dd06-03eb-41fa-a3a3-35d2947b4432 /tidb/cdc/default/default/changefeed/info/ddl-manager {"upstream-id":7489988224941036897,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2025-04-06T08:19:53.003671318+08:00","start-ts":457152605159424004,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457152605185376262} /tidb/cdc/default/default/changefeed/status/ddl-manager {"checkpoint-ts":457152605854105601,"min-table-barrier-ts":457152605854105601,"admin-job-type":0} /tidb/cdc/default/default/task/position/2319dd06-03eb-41fa-a3a3-35d2947b4432/ddl-manager {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489988224941036897 {"id":7489988224941036897,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-manager {UpstreamID:7489988224941036897 Namespace:default ID:ddl-manager SinkURI:mysql://root@127.0.0.1:3306/ CreateTime:2025-04-06 08:19:53.003671318 +0800 CST StartTs:457152605159424004 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00441a240 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457152605185376262} {CheckpointTs:457152605854105601 MinTableBarrierTs:457152605854105601 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2319dd06-03eb-41fa-a3a3-35d2947b4432 {"id":"2319dd06-03eb-41fa-a3a3-35d2947b4432","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898815} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319608779cf722 2319dd06-03eb-41fa-a3a3-35d2947b4432 /tidb/cdc/default/default/changefeed/info/ddl-manager {"upstream-id":7489988224941036897,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2025-04-06T08:19:53.003671318+08:00","start-ts":457152605159424004,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457152605185376262} /tidb/cdc/default/default/changefeed/status/ddl-manager {"checkpoint-ts":457152605854105601,"min-table-barrier-ts":457152605854105601,"admin-job-type":0} /tidb/cdc/default/default/task/position/2319dd06-03eb-41fa-a3a3-35d2947b4432/ddl-manager {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489988224941036897 {"id":7489988224941036897,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-manager {UpstreamID:7489988224941036897 Namespace:default ID:ddl-manager SinkURI:mysql://root@127.0.0.1:3306/ CreateTime:2025-04-06 08:19:53.003671318 +0800 CST StartTs:457152605159424004 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00441a240 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457152605185376262} {CheckpointTs:457152605854105601 MinTableBarrierTs:457152605854105601 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2319dd06-03eb-41fa-a3a3-35d2947b4432 {"id":"2319dd06-03eb-41fa-a3a3-35d2947b4432","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898815} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319608779cf722 2319dd06-03eb-41fa-a3a3-35d2947b4432 /tidb/cdc/default/default/changefeed/info/ddl-manager {"upstream-id":7489988224941036897,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2025-04-06T08:19:53.003671318+08:00","start-ts":457152605159424004,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457152605185376262} /tidb/cdc/default/default/changefeed/status/ddl-manager {"checkpoint-ts":457152605854105601,"min-table-barrier-ts":457152605854105601,"admin-job-type":0} /tidb/cdc/default/default/task/position/2319dd06-03eb-41fa-a3a3-35d2947b4432/ddl-manager {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489988224941036897 {"id":7489988224941036897,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Sun Apr 6 08:20:20 CST 2025] <<<<<< START cdc server in ddl_manager case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ExecuteDDLSlowly=return(true)' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_manager.54625464.out server --log-file /tmp/tidb_cdc_test/ddl_manager/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_manager/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > start tidb cluster in /tmp/tidb_cdc_test/multi_capture Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/changefeed_error Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/http_api_tls/run.sh using Sink-Type: mysql... <<================= WARNING: Running pip install with root privileges is generally not a good idea. Try `__main__.py install --user` instead. Requirement already up-to-date: requests==2.26.0 in /usr/local/lib/python3.6/site-packages Requirement already up-to-date: idna<4,>=2.5; python_version >= "3" in /usr/local/lib/python3.6/site-packages (from requests==2.26.0) Requirement already up-to-date: charset-normalizer~=2.0.0; python_version >= "3" in /usr/local/lib/python3.6/site-packages (from requests==2.26.0) Requirement already up-to-date: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.6/site-packages (from requests==2.26.0) Requirement already up-to-date: certifi>=2017.4.17 in /usr/local/lib/python3.6/site-packages (from requests==2.26.0) The 1 times to try to start tidb cluster... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/partition_table/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release < HTTP/1.1 200 OK < Date: Sun, 06 Apr 2025 00:20:25 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/ddl-manager {UpstreamID:7489988224941036897 Namespace:default ID:ddl-manager SinkURI:mysql://root@127.0.0.1:3306/ CreateTime:2025-04-06 08:19:53.003671318 +0800 CST StartTs:457152605159424004 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00441a240 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457152605185376262} {CheckpointTs:457152605854105601 MinTableBarrierTs:457152605854105601 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2319dd06-03eb-41fa-a3a3-35d2947b4432 {"id":"2319dd06-03eb-41fa-a3a3-35d2947b4432","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898815} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319608779cf722 2319dd06-03eb-41fa-a3a3-35d2947b4432 /tidb/cdc/default/default/changefeed/info/ddl-manager {"upstream-id":7489988224941036897,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2025-04-06T08:19:53.003671318+08:00","start-ts":457152605159424004,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457152605185376262} /tidb/cdc/default/default/changefeed/status/ddl-manager {"checkpoint-ts":457152605984915463,"min-table-barrier-ts":457152605984915463,"admin-job-type":0} /tidb/cdc/default/default/task/position/2319dd06-03eb-41fa-a3a3-35d2947b4432/ddl-manager {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489988224941036897 {"id":7489988224941036897,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-manager {UpstreamID:7489988224941036897 Namespace:default ID:ddl-manager SinkURI:mysql://root@127.0.0.1:3306/ CreateTime:2025-04-06 08:19:53.003671318 +0800 CST StartTs:457152605159424004 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00441a240 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457152605185376262} {CheckpointTs:457152605854105601 MinTableBarrierTs:457152605854105601 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2319dd06-03eb-41fa-a3a3-35d2947b4432 {"id":"2319dd06-03eb-41fa-a3a3-35d2947b4432","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898815} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319608779cf722 2319dd06-03eb-41fa-a3a3-35d2947b4432 /tidb/cdc/default/default/changefeed/info/ddl-manager {"upstream-id":7489988224941036897,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2025-04-06T08:19:53.003671318+08:00","start-ts":457152605159424004,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457152605185376262} /tidb/cdc/default/default/changefeed/status/ddl-manager {"checkpoint-ts":457152605984915463,"min-table-barrier-ts":457152605984915463,"admin-job-type":0} /tidb/cdc/default/default/task/position/2319dd06-03eb-41fa-a3a3-35d2947b4432/ddl-manager {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489988224941036897 + grep -q 'failed to get info:' {"id":7489988224941036897,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-manager {UpstreamID:7489988224941036897 Namespace:default ID:ddl-manager SinkURI:mysql://root@127.0.0.1:3306/ CreateTime:2025-04-06 08:19:53.003671318 +0800 CST StartTs:457152605159424004 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00441a240 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457152605185376262} {CheckpointTs:457152605854105601 MinTableBarrierTs:457152605854105601 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2319dd06-03eb-41fa-a3a3-35d2947b4432 {"id":"2319dd06-03eb-41fa-a3a3-35d2947b4432","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898815} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319608779cf722 2319dd06-03eb-41fa-a3a3-35d2947b4432 /tidb/cdc/default/default/changefeed/info/ddl-manager {"upstream-id":7489988224941036897,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2025-04-06T08:19:53.003671318+08:00","start-ts":457152605159424004,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457152605185376262} /tidb/cdc/default/default/changefeed/status/ddl-manager {"checkpoint-ts":457152605984915463,"min-table-barrier-ts":457152605984915463,"admin-job-type":0} /tidb/cdc/default/default/task/position/2319dd06-03eb-41fa-a3a3-35d2947b4432/ddl-manager {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489988224941036897 {"id":7489988224941036897,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x table ddl_manager.finish_mark not exists for 1-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 2-th check, retry later Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Cache restored successfully (ws/jenkins-pingcap-tiflow-periodics_cdc_integration_mysql_test-234/tiflow-cdc) 4028702720 bytes in 8.02 secs (502116063 bytes/sec) [Pipeline] { [Pipeline] cache start tidb cluster in /tmp/tidb_cdc_test/http_api_tls Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... start tidb cluster in /tmp/tidb_cdc_test/partition_table Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489988408127107986 TiDB Cluster ID. tikv_gc_leader_uuid 65821e16f500006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-9-92vm1, pid:9517, start at 2025-04-06 08:20:27.74312569 +0800 CST m=+2.361402987 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:22:27.756 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:20:27.732 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:10:27.732 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489988408127107986 TiDB Cluster ID. tikv_gc_leader_uuid 65821e16f500006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-9-92vm1, pid:9517, start at 2025-04-06 08:20:27.74312569 +0800 CST m=+2.361402987 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:22:27.756 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:20:27.732 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:10:27.732 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489988409383529073 TiDB Cluster ID. tikv_gc_leader_uuid 65821e16f48000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-9-92vm1, pid:9595, start at 2025-04-06 08:20:27.770264966 +0800 CST m=+2.322200085 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:22:27.779 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:20:27.774 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:10:27.774 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/syncpoint/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/syncpoint/tiflash/log/error.log arg matches is ArgMatches { args: {"pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/syncpoint/tiflash/db/proxy"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/syncpoint/tiflash-proxy.toml"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/syncpoint/tiflash/log/proxy.log"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 3-th check, retry later Verifying downstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 4-th check, retry later + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.syncpoint.cli.10868.out cli tso query --pd=http://127.0.0.1:2379 Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489988418452591948 TiDB Cluster ID. tikv_gc_leader_uuid 65821e196b0000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-m-cm2pq, pid:4075, start at 2025-04-06 08:20:30.29033643 +0800 CST m=+2.284777472 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:22:30.335 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:20:30.301 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:10:30.301 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489988418452591948 TiDB Cluster ID. tikv_gc_leader_uuid 65821e196b0000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-m-cm2pq, pid:4075, start at 2025-04-06 08:20:30.29033643 +0800 CST m=+2.284777472 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:22:30.335 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:20:30.301 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:10:30.301 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489988419102463629 TiDB Cluster ID. tikv_gc_leader_uuid 65821e197e80007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-m-cm2pq, pid:4137, start at 2025-04-06 08:20:30.341093472 +0800 CST m=+2.277092161 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:22:30.348 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:20:30.350 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:10:30.350 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/changefeed_error/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/changefeed_error/tiflash/log/error.log arg matches is ArgMatches { args: {"log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/changefeed_error/tiflash/log/proxy.log"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/changefeed_error/tiflash-proxy.toml"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/changefeed_error/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + set +x + tso='457152615476625409 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 457152615476625409 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Sun Apr 6 08:20:33 CST 2025] <<<<<< START cdc server in syncpoint case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/DDLSinkInitializeSlowly=return(true)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.syncpoint.1090710909.out server --log-file /tmp/tidb_cdc_test/syncpoint/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/syncpoint/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489988426194103460 TiDB Cluster ID. tikv_gc_leader_uuid 65821e1af240004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-d-7dnbd, pid:7097, start at 2025-04-06 08:20:31.820392607 +0800 CST m=+2.556056810 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:22:31.864 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:20:31.866 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:10:31.866 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489988426194103460 TiDB Cluster ID. tikv_gc_leader_uuid 65821e1af240004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-d-7dnbd, pid:7097, start at 2025-04-06 08:20:31.820392607 +0800 CST m=+2.556056810 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:22:31.864 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:20:31.866 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:10:31.866 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489988417009980969 TiDB Cluster ID. tikv_gc_leader_uuid 65821e1af1c0006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-d-7dnbd, pid:7167, start at 2025-04-06 08:20:31.859200071 +0800 CST m=+2.537060910 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:22:31.867 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:20:31.865 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:10:31.865 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/multi_capture/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/multi_capture/tiflash/log/error.log arg matches is ArgMatches { args: {"log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/multi_capture/tiflash/log/proxy.log"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/multi_capture/tiflash-proxy.toml"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/multi_capture/tiflash/db/proxy"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table ddl_manager.finish_mark not exists for 5-th check, retry later Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.5388.out cli tso query --pd=http://127.0.0.1:2379 Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_capture.cli.8408.out cli tso query --pd=http://127.0.0.1:2379 table ddl_manager.finish_mark not exists for 6-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x + tso='457152616123858945 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 457152616123858945 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 06 Apr 2025 00:20:36 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ad7c7d75-31c4-4744-a9b6-2d2bbbecc6cc {"id":"ad7c7d75-31c4-4744-a9b6-2d2bbbecc6cc","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898834} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319608783eb3ee ad7c7d75-31c4-4744-a9b6-2d2bbbecc6cc /tidb/cdc/default/default/upstream/7489988408127107986 {"id":7489988408127107986,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ad7c7d75-31c4-4744-a9b6-2d2bbbecc6cc {"id":"ad7c7d75-31c4-4744-a9b6-2d2bbbecc6cc","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898834} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319608783eb3ee ad7c7d75-31c4-4744-a9b6-2d2bbbecc6cc /tidb/cdc/default/default/upstream/7489988408127107986 {"id":7489988408127107986,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ad7c7d75-31c4-4744-a9b6-2d2bbbecc6cc {"id":"ad7c7d75-31c4-4744-a9b6-2d2bbbecc6cc","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898834} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319608783eb3ee ad7c7d75-31c4-4744-a9b6-2d2bbbecc6cc /tidb/cdc/default/default/upstream/7489988408127107986 {"id":7489988408127107986,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.syncpoint.cli.10966.out cli changefeed create --start-ts=457152615476625409 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --config=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/syncpoint/conf/changefeed.toml Create changefeed successfully! ID: 4434f91c-3e84-4463-b81f-e77de8423ecc Info: {"upstream_id":7489988408127107986,"namespace":"default","id":"4434f91c-3e84-4463-b81f-e77de8423ecc","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2025-04-06T08:20:37.200653512+08:00","start_ts":457152615476625409,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":true,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":30000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457152615476625409,"checkpoint_ts":457152615476625409,"checkpoint_time":"2025-04-06 08:20:32.232"} PASS ***************** properties ***************** "updateproportion"="0" "mysql.host"="127.0.0.1" "mysql.db"="changefeed_error" "threadcount"="4" "operationcount"="0" "mysql.port"="4000" "readproportion"="0" "recordcount"="20" "mysql.user"="root" "scanproportion"="0" "readallfields"="true" "workload"="core" "dotransactions"="false" "insertproportion"="0" "requestdistribution"="uniform" ********************************************** Run finished, takes 12.530812ms INSERT - Takes(s): 0.0, Count: 20, OPS: 3124.5, Avg(us): 2364, Min(us): 1167, Max(us): 6085, 95th(us): 7000, 99th(us): 7000 [Sun Apr 6 08:20:37 CST 2025] <<<<<< START cdc server in changefeed_error case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/NewChangefeedNoRetryError=1*return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.54545456.out server --log-file /tmp/tidb_cdc_test/changefeed_error/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_error/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x + tso='457152616468578305 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 457152616468578305 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x ***************** properties ***************** "updateproportion"="0" "operationcount"="0" "mysql.port"="4000" "threadcount"="2" "readallfields"="true" "requestdistribution"="uniform" "mysql.host"="127.0.0.1" "dotransactions"="false" "scanproportion"="0" "workload"="core" "insertproportion"="0" "mysql.db"="multi_capture_1" "mysql.user"="root" "recordcount"="10" "readproportion"="0" ********************************************** Run finished, takes 11.159709ms INSERT - Takes(s): 0.0, Count: 10, OPS: 1724.5, Avg(us): 2108, Min(us): 1074, Max(us): 5227, 95th(us): 6000, 99th(us): 6000 ***************** properties ***************** "operationcount"="0" "readallfields"="true" "dotransactions"="false" "requestdistribution"="uniform" "mysql.db"="multi_capture_2" "mysql.host"="127.0.0.1" "recordcount"="10" "threadcount"="2" "readproportion"="0" "scanproportion"="0" "mysql.port"="4000" "mysql.user"="root" "insertproportion"="0" "workload"="core" "updateproportion"="0" ********************************************** Run finished, takes 10.10629ms INSERT - Takes(s): 0.0, Count: 10, OPS: 1771.1, Avg(us): 1946, Min(us): 1154, Max(us): 4346, 95th(us): 5000, 99th(us): 5000 table ddl_manager.finish_mark not exists for 7-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489988454624462014 TiDB Cluster ID. tikv_gc_leader_uuid 65821e2052c0001 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-8-31x9h, pid:4439, start at 2025-04-06 08:20:37.323852337 +0800 CST m=+2.018711114 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:22:37.331 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:20:37.323 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:10:37.323 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ***************** properties ***************** "workload"="core" "dotransactions"="false" "insertproportion"="0" "mysql.db"="multi_capture_3" "scanproportion"="0" "readallfields"="true" "mysql.host"="127.0.0.1" "mysql.port"="4000" "readproportion"="0" "threadcount"="2" "mysql.user"="root" "updateproportion"="0" "recordcount"="10" "operationcount"="0" "requestdistribution"="uniform" ********************************************** Run finished, takes 9.789686ms INSERT - Takes(s): 0.0, Count: 10, OPS: 1900.2, Avg(us): 1859, Min(us): 1145, Max(us): 4464, 95th(us): 5000, 99th(us): 5000 ***************** properties ***************** "readproportion"="0" "insertproportion"="0" "readallfields"="true" "updateproportion"="0" "mysql.port"="4000" "dotransactions"="false" "operationcount"="0" "threadcount"="2" "mysql.host"="127.0.0.1" "scanproportion"="0" "recordcount"="10" "mysql.user"="root" "mysql.db"="multi_capture_4" "workload"="core" "requestdistribution"="uniform" ********************************************** Run finished, takes 8.804968ms INSERT - Takes(s): 0.0, Count: 10, OPS: 1880.0, Avg(us): 1679, Min(us): 1029, Max(us): 3395, 95th(us): 4000, 99th(us): 4000 [Sun Apr 6 08:20:37 CST 2025] <<<<<< START cdc server in multi_capture case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_capture.85318533.out server --log-file /tmp/tidb_cdc_test/multi_capture/cdc1.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_capture/cdc_data1 --cluster-id default --addr 127.0.0.1:8301 + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489988441334275052 TiDB Cluster ID. tikv_gc_leader_uuid 65821e1f9340005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-k-bx9s4, pid:9451, start at 2025-04-06 08:20:36.564375657 +0800 CST m=+2.482149795 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:22:36.572 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:20:36.557 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:10:36.557 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489988441334275052 TiDB Cluster ID. tikv_gc_leader_uuid 65821e1f9340005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-k-bx9s4, pid:9451, start at 2025-04-06 08:20:36.564375657 +0800 CST m=+2.482149795 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:22:36.572 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:20:36.557 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:10:36.557 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489988443427808532 TiDB Cluster ID. tikv_gc_leader_uuid 65821e1fac40003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-k-bx9s4, pid:9527, start at 2025-04-06 08:20:36.659832096 +0800 CST m=+2.488876948 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:22:36.668 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:20:36.657 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:10:36.657 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/partition_table/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/partition_table/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/partition_table/tiflash-proxy.toml"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/partition_table/tiflash/log/proxy.log"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/partition_table/tiflash/db/proxy"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + set +x ***************** properties ***************** "mysql.host"="127.0.0.1" "readallfields"="true" "insertproportion"="0" "threadcount"="2" "updateproportion"="0" "operationcount"="0" "mysql.port"="4000" "mysql.user"="root" "dotransactions"="false" "recordcount"="10" "readproportion"="0" "requestdistribution"="uniform" "scanproportion"="0" "workload"="core" "mysql.db"="testSync" ********************************************** Run finished, takes 12.2562ms INSERT - Takes(s): 0.0, Count: 10, OPS: 1297.7, Avg(us): 2150, Min(us): 1168, Max(us): 6422, 95th(us): 7000, 99th(us): 7000 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489988454624462014 TiDB Cluster ID. tikv_gc_leader_uuid 65821e2052c0001 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-8-31x9h, pid:4439, start at 2025-04-06 08:20:37.323852337 +0800 CST m=+2.018711114 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:22:37.331 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:20:37.323 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:10:37.323 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489988440611463034 TiDB Cluster ID. tikv_gc_leader_uuid 65821e20680000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-8-31x9h, pid:4521, start at 2025-04-06 08:20:37.44114008 +0800 CST m=+2.078319931 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:22:37.448 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:20:37.408 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:10:37.408 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/http_api_tls/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/http_api_tls/tiflash/log/error.log arg matches is ArgMatches { args: {"memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/http_api_tls/tiflash-proxy.toml"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/http_api_tls/tiflash/db/proxy"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/http_api_tls/tiflash/log/proxy.log"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Cache restored successfully (ws/jenkins-pingcap-tiflow-periodics_cdc_integration_mysql_test-234/tiflow-cdc) 4028702720 bytes in 8.38 secs (480879573 bytes/sec) [Pipeline] { [Pipeline] cache table ddl_manager.finish_mark not exists for 8-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 06 Apr 2025 00:20:40 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/fa9b673a-72d5-464a-816e-71a04753df77 {"id":"fa9b673a-72d5-464a-816e-71a04753df77","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898837} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087848effc fa9b673a-72d5-464a-816e-71a04753df77 /tidb/cdc/default/default/upstream/7489988418452591948 {"id":7489988418452591948,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/fa9b673a-72d5-464a-816e-71a04753df77 {"id":"fa9b673a-72d5-464a-816e-71a04753df77","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898837} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087848effc fa9b673a-72d5-464a-816e-71a04753df77 /tidb/cdc/default/default/upstream/7489988418452591948 {"id":7489988418452591948,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/fa9b673a-72d5-464a-816e-71a04753df77 {"id":"fa9b673a-72d5-464a-816e-71a04753df77","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898837} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087848effc fa9b673a-72d5-464a-816e-71a04753df77 /tidb/cdc/default/default/upstream/7489988418452591948 {"id":7489988418452591948,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.5517.out cli changefeed create --start-ts=457152616123858945 '--sink-uri=mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1' -c changefeed-error + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.partition_table.cli.10770.out cli tso query --pd=http://127.0.0.1:2379 Create changefeed successfully! ID: changefeed-error Info: {"upstream_id":7489988418452591948,"namespace":"default","id":"changefeed-error","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create_time":"2025-04-06T08:20:40.75154244+08:00","start_ts":457152616123858945,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457152616123858945,"checkpoint_ts":457152616123858945,"checkpoint_time":"2025-04-06 08:20:34.701"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 06 Apr 2025 00:20:40 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c26c60f4-ff52-4550-a5af-3721f8c72d4a {"id":"c26c60f4-ff52-4550-a5af-3721f8c72d4a","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898838} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087849f826 c26c60f4-ff52-4550-a5af-3721f8c72d4a /tidb/cdc/default/default/upstream/7489988426194103460 {"id":7489988426194103460,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c26c60f4-ff52-4550-a5af-3721f8c72d4a {"id":"c26c60f4-ff52-4550-a5af-3721f8c72d4a","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898838} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087849f826 c26c60f4-ff52-4550-a5af-3721f8c72d4a /tidb/cdc/default/default/upstream/7489988426194103460 {"id":7489988426194103460,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c26c60f4-ff52-4550-a5af-3721f8c72d4a {"id":"c26c60f4-ff52-4550-a5af-3721f8c72d4a","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898838} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087849f826 c26c60f4-ff52-4550-a5af-3721f8c72d4a /tidb/cdc/default/default/upstream/7489988426194103460 {"id":7489988426194103460,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Sun Apr 6 08:20:40 CST 2025] <<<<<< START cdc server in multi_capture case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_capture.85828584.out server --log-file /tmp/tidb_cdc_test/multi_capture/cdc2.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_capture/cdc_data2 --cluster-id default --addr 127.0.0.1:8302 ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8302; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 The 1 times to try to start tls tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/http_api_tls Starting TLS PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 table ddl_manager.finish_mark not exists for 9-th check, retry later + set +x check_changefeed_state http://127.0.0.1:2379 changefeed-error failed [CDC:ErrStartTsBeforeGC] + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-error + expected_state=failed + error_msg='[CDC:ErrStartTsBeforeGC]' + tls_dir='[CDC:ErrStartTsBeforeGC]' + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-error -s + info='{ "upstream_id": 7489988418452591948, "namespace": "default", "id": "changefeed-error", "state": "failed", "checkpoint_tso": 457152616123858945, "checkpoint_time": "2025-04-06 08:20:34.701", "error": { "time": "2025-04-06T08:20:40.857329529+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrStartTsBeforeGC", "message": "[CDC:ErrStartTsBeforeGC]fail to create or maintain changefeed because start-ts 457152616123858645 is earlier than or equal to GC safepoint at 457152616123858945" } }' + echo '{ "upstream_id": 7489988418452591948, "namespace": "default", "id": "changefeed-error", "state": "failed", "checkpoint_tso": 457152616123858945, "checkpoint_time": "2025-04-06 08:20:34.701", "error": { "time": "2025-04-06T08:20:40.857329529+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrStartTsBeforeGC", "message": "[CDC:ErrStartTsBeforeGC]fail to create or maintain changefeed because start-ts 457152616123858645 is earlier than or equal to GC safepoint at 457152616123858945" } }' { "upstream_id": 7489988418452591948, "namespace": "default", "id": "changefeed-error", "state": "failed", "checkpoint_tso": 457152616123858945, "checkpoint_time": "2025-04-06 08:20:34.701", "error": { "time": "2025-04-06T08:20:40.857329529+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrStartTsBeforeGC", "message": "[CDC:ErrStartTsBeforeGC]fail to create or maintain changefeed because start-ts 457152616123858645 is earlier than or equal to GC safepoint at 457152616123858945" } } ++ echo '{' '"upstream_id":' 7489988418452591948, '"namespace":' '"default",' '"id":' '"changefeed-error",' '"state":' '"failed",' '"checkpoint_tso":' 457152616123858945, '"checkpoint_time":' '"2025-04-06' '08:20:34.701",' '"error":' '{' '"time":' '"2025-04-06T08:20:40.857329529+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrStartTsBeforeGC",' '"message":' '"[CDC:ErrStartTsBeforeGC]fail' to create or maintain changefeed because start-ts 457152616123858645 is earlier than or equal to GC safepoint at '457152616123858945"' '}' '}' ++ jq -r .state + state=failed + [[ ! failed == \f\a\i\l\e\d ]] ++ echo '{' '"upstream_id":' 7489988418452591948, '"namespace":' '"default",' '"id":' '"changefeed-error",' '"state":' '"failed",' '"checkpoint_tso":' 457152616123858945, '"checkpoint_time":' '"2025-04-06' '08:20:34.701",' '"error":' '{' '"time":' '"2025-04-06T08:20:40.857329529+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrStartTsBeforeGC",' '"message":' '"[CDC:ErrStartTsBeforeGC]fail' to create or maintain changefeed because start-ts 457152616123858645 is earlier than or equal to GC safepoint at '457152616123858945"' '}' '}' ++ jq -r .error.message + message='[CDC:ErrStartTsBeforeGC]fail to create or maintain changefeed because start-ts 457152616123858645 is earlier than or equal to GC safepoint at 457152616123858945' + [[ ! [CDC:ErrStartTsBeforeGC]fail to create or maintain changefeed because start-ts 457152616123858645 is earlier than or equal to GC safepoint at 457152616123858945 =~ \[CDC:ErrStartTsBeforeGC] ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.5589.out cli changefeed resume -c changefeed-error + set +x + tso='457152617723199489 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 457152617723199489 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Sun Apr 6 08:20:42 CST 2025] <<<<<< START cdc server in partition_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.partition_table.1080110803.out server --log-file /tmp/tidb_cdc_test/partition_table/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/partition_table/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 PASS coverage: 2.1% of statements in github.com/pingcap/tiflow/... table ddl_manager.finish_mark not exists for 10-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8302 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8302 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 06 Apr 2025 00:20:43 GMT < Content-Length: 1281 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1df057d7-ce67-4a81-a7b9-80e72d8f8693 {"id":"1df057d7-ce67-4a81-a7b9-80e72d8f8693","address":"127.0.0.1:8302","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898841} /tidb/cdc/default/__cdc_meta__/capture/c26c60f4-ff52-4550-a5af-3721f8c72d4a {"id":"c26c60f4-ff52-4550-a5af-3721f8c72d4a","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898838} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087849f826 c26c60f4-ff52-4550-a5af-3721f8c72d4a /tidb/cdc/default/__cdc_meta__/owner/223196087849f85b 1df057d7-ce67-4a81-a7b9-80e72d8f8693 /tidb/cdc/default/default/upstream/7489988426194103460 {"id":7489988426194103460,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1df057d7-ce67-4a81-a7b9-80e72d8f8693 {"id":"1df057d7-ce67-4a81-a7b9-80e72d8f8693","address":"127.0.0.1:8302","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898841} /tidb/cdc/default/__cdc_meta__/capture/c26c60f4-ff52-4550-a5af-3721f8c72d4a {"id":"c26c60f4-ff52-4550-a5af-3721f8c72d4a","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898838} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087849f826 c26c60f4-ff52-4550-a5af-3721f8c72d4a /tidb/cdc/default/__cdc_meta__/owner/223196087849f85b 1df057d7-ce67-4a81-a7b9-80e72d8f8693 /tidb/cdc/default/default/upstream/7489988426194103460 {"id":7489988426194103460,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1df057d7-ce67-4a81-a7b9-80e72d8f8693 {"id":"1df057d7-ce67-4a81-a7b9-80e72d8f8693","address":"127.0.0.1:8302","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898841} /tidb/cdc/default/__cdc_meta__/capture/c26c60f4-ff52-4550-a5af-3721f8c72d4a {"id":"c26c60f4-ff52-4550-a5af-3721f8c72d4a","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898838} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087849f826 c26c60f4-ff52-4550-a5af-3721f8c72d4a /tidb/cdc/default/__cdc_meta__/owner/223196087849f85b 1df057d7-ce67-4a81-a7b9-80e72d8f8693 /tidb/cdc/default/default/upstream/7489988426194103460 {"id":7489988426194103460,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Sun Apr 6 08:20:43 CST 2025] <<<<<< START cdc server in multi_capture case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8303/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_capture.86418643.out server --log-file /tmp/tidb_cdc_test/multi_capture/cdc3.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_capture/cdc_data3 --cluster-id default --addr 127.0.0.1:8303 ++ curl -vsL --max-time 20 http://127.0.0.1:8303/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8303 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8303; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + set +x table changefeed_error.usertable exists check diff failed 1-th time, retry later Starting TLS TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 06 Apr 2025 00:20:45 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f0662544-d9b7-48b6-a16c-04c9d7b7bdff {"id":"f0662544-d9b7-48b6-a16c-04c9d7b7bdff","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898842} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319608786090eb f0662544-d9b7-48b6-a16c-04c9d7b7bdff /tidb/cdc/default/default/upstream/7489988441334275052 {"id":7489988441334275052,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f0662544-d9b7-48b6-a16c-04c9d7b7bdff {"id":"f0662544-d9b7-48b6-a16c-04c9d7b7bdff","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898842} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319608786090eb f0662544-d9b7-48b6-a16c-04c9d7b7bdff /tidb/cdc/default/default/upstream/7489988441334275052 {"id":7489988441334275052,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f0662544-d9b7-48b6-a16c-04c9d7b7bdff {"id":"f0662544-d9b7-48b6-a16c-04c9d7b7bdff","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898842} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319608786090eb f0662544-d9b7-48b6-a16c-04c9d7b7bdff /tidb/cdc/default/default/upstream/7489988441334275052 {"id":7489988441334275052,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.partition_table.cli.10860.out cli changefeed create --start-ts=457152617723199489 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ table ddl_manager.finish_mark not exists for 11-th check, retry later Create changefeed successfully! ID: d6165229-76a6-4b43-b0f4-e21ddb271c5f Info: {"upstream_id":7489988441334275052,"namespace":"default","id":"d6165229-76a6-4b43-b0f4-e21ddb271c5f","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-06T08:20:45.820268573+08:00","start_ts":457152617723199489,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457152617723199489,"checkpoint_ts":457152617723199489,"checkpoint_time":"2025-04-06 08:20:40.802"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... Starting TLS TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying TLS TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8303/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8303 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8303 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8303 > Accept: */* > check diff successfully ***************** properties ***************** "requestdistribution"="uniform" "readallfields"="true" "workload"="core" "mysql.host"="127.0.0.1" "dotransactions"="false" "mysql.port"="4000" "mysql.db"="changefeed_error" "recordcount"="20" "operationcount"="0" "mysql.user"="root" "threadcount"="4" "readproportion"="0" "insertproportion"="0" "updateproportion"="0" "scanproportion"="0" ********************************************** Run finished, takes 4.87923ms INSERT - Takes(s): 0.0, Count: 20, OPS: 6297.7, Avg(us): 869, Min(us): 423, Max(us): 2202, 95th(us): 3000, 99th(us): 3000 check diff successfully {"id":"fa9b673a-72d5-464a-816e-71a04753df77","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898837} check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/capture' 'capture' + key_prefix=/tidb/cdc/default/__cdc_meta__/capture + message=capture ++ etcdctl get /tidb/cdc/default/__cdc_meta__/capture --prefix --keys-only + info=/tidb/cdc/default/__cdc_meta__/capture/fa9b673a-72d5-464a-816e-71a04753df77 + [[ /tidb/cdc/default/__cdc_meta__/capture/fa9b673a-72d5-464a-816e-71a04753df77 =~ capture ]] + echo 'capture contains in etcd /tidb/cdc/default/__cdc_meta__/capture/fa9b673a-72d5-464a-816e-71a04753df77' capture contains in etcd /tidb/cdc/default/__cdc_meta__/capture/fa9b673a-72d5-464a-816e-71a04753df77 + echo 'check failed' check failed + exit 1 run task failed 1-th time, retry later < HTTP/1.1 200 OK < Date: Sun, 06 Apr 2025 00:20:47 GMT < Content-Length: 1765 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1df057d7-ce67-4a81-a7b9-80e72d8f8693 {"id":"1df057d7-ce67-4a81-a7b9-80e72d8f8693","address":"127.0.0.1:8302","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898841} /tidb/cdc/default/__cdc_meta__/capture/7ce7522e-3115-4c10-b46e-137a7ab9fdf0 {"id":"7ce7522e-3115-4c10-b46e-137a7ab9fdf0","address":"127.0.0.1:8303","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898844} /tidb/cdc/default/__cdc_meta__/capture/c26c60f4-ff52-4550-a5af-3721f8c72d4a {"id":"c26c60f4-ff52-4550-a5af-3721f8c72d4a","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898838} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087849f826 c26c60f4-ff52-4550-a5af-3721f8c72d4a /tidb/cdc/default/__cdc_meta__/owner/223196087849f85b 1df057d7-ce67-4a81-a7b9-80e72d8f8693 /tidb/cdc/default/__cdc_meta__/owner/223196087849f87c 7ce7522e-3115-4c10-b46e-137a7ab9fdf0 /tidb/cdc/default/default/upstream/7489988426194103460 {"id":7489988426194103460,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1df057d7-ce67-4a81-a7b9-80e72d8f8693 {"id":"1df057d7-ce67-4a81-a7b9-80e72d8f8693","address":"127.0.0.1:8302","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898841} /tidb/cdc/default/__cdc_meta__/capture/7ce7522e-3115-4c10-b46e-137a7ab9fdf0 {"id":"7ce7522e-3115-4c10-b46e-137a7ab9fdf0","address":"127.0.0.1:8303","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898844} /tidb/cdc/default/__cdc_meta__/capture/c26c60f4-ff52-4550-a5af-3721f8c72d4a {"id":"c26c60f4-ff52-4550-a5af-3721f8c72d4a","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898838} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087849f826 c26c60f4-ff52-4550-a5af-3721f8c72d4a /tidb/cdc/default/__cdc_meta__/owner/223196087849f85b 1df057d7-ce67-4a81-a7b9-80e72d8f8693 /tidb/cdc/default/__cdc_meta__/owner/223196087849f87c 7ce7522e-3115-4c10-b46e-137a7ab9fdf0 /tidb/cdc/default/default/upstream/7489988426194103460 {"id":7489988426194103460,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1df057d7-ce67-4a81-a7b9-80e72d8f8693 {"id":"1df057d7-ce67-4a81-a7b9-80e72d8f8693","address":"127.0.0.1:8302","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898841} /tidb/cdc/default/__cdc_meta__/capture/7ce7522e-3115-4c10-b46e-137a7ab9fdf0 {"id":"7ce7522e-3115-4c10-b46e-137a7ab9fdf0","address":"127.0.0.1:8303","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898844} /tidb/cdc/default/__cdc_meta__/capture/c26c60f4-ff52-4550-a5af-3721f8c72d4a {"id":"c26c60f4-ff52-4550-a5af-3721f8c72d4a","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898838} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087849f826 c26c60f4-ff52-4550-a5af-3721f8c72d4a /tidb/cdc/default/__cdc_meta__/owner/223196087849f85b 1df057d7-ce67-4a81-a7b9-80e72d8f8693 /tidb/cdc/default/__cdc_meta__/owner/223196087849f87c 7ce7522e-3115-4c10-b46e-137a7ab9fdf0 /tidb/cdc/default/default/upstream/7489988426194103460 {"id":7489988426194103460,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_capture.cli.8707.out cli changefeed create --start-ts=457152616468578305 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ --server=127.0.0.1:8301 + set +x Create changefeed successfully! ID: 0dbbe661-df06-417d-ac2e-1f5145557b7b Info: {"upstream_id":7489988426194103460,"namespace":"default","id":"0dbbe661-df06-417d-ac2e-1f5145557b7b","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-06T08:20:47.49790478+08:00","start_ts":457152616468578305,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457152616468578305,"checkpoint_ts":457152616468578305,"checkpoint_time":"2025-04-06 08:20:36.016"} PASS table ddl_manager.finish_mark not exists for 12-th check, retry later coverage: 2.4% of statements in github.com/pingcap/tiflow/... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489988500234899460 TiDB Cluster ID. tikv_gc_leader_uuid 65821e2ac1c000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-8-31x9h, pid:6065, start at 2025-04-06 08:20:48.051335697 +0800 CST m=+1.448903740 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:22:48.057 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:20:48.056 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:10:48.056 +0800 All versions after safe point can be accessed. (DO NOT EDIT) [Sun Apr 6 08:20:48 CST 2025] <<<<<< START cdc server in http_api_tls case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates ']' + curl_status_cmd='curl --cacert /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client.pem --key /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client-key.pem --user ticdc:ticdc_secret -vsL --max-time 20 https://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.http_api_tls.61116113.out server --log-file /tmp/tidb_cdc_test/http_api_tls/cdc_http_api_tls_tls1.log --log-level debug --data-dir /tmp/tidb_cdc_test/http_api_tls/cdc_data_http_api_tls_tls1 --cluster-id default --config /tmp/tidb_cdc_test/http_api_tls/server.toml --ca /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/server.pem --key /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/server-key.pem --cert-allowed-cn client --addr 127.0.0.1:8300 --pd https://127.0.0.1:2579 + (( i = 0 )) + (( i <= 50 )) ++ curl --cacert /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client.pem --key /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client-key.pem --user ticdc:ticdc_secret -vsL --max-time 20 https://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/capture' 'capture' + key_prefix=/tidb/cdc/default/__cdc_meta__/capture + message=capture ++ etcdctl get /tidb/cdc/default/__cdc_meta__/capture --prefix --keys-only + info=/tidb/cdc/default/__cdc_meta__/capture/fa9b673a-72d5-464a-816e-71a04753df77 + [[ /tidb/cdc/default/__cdc_meta__/capture/fa9b673a-72d5-464a-816e-71a04753df77 =~ capture ]] + echo 'capture contains in etcd /tidb/cdc/default/__cdc_meta__/capture/fa9b673a-72d5-464a-816e-71a04753df77' capture contains in etcd /tidb/cdc/default/__cdc_meta__/capture/fa9b673a-72d5-464a-816e-71a04753df77 + echo 'check failed' check failed + exit 1 run task failed 2-th time, retry later + set +x table multi_capture_1.usertable not exists for 1-th check, retry later table ddl_manager.finish_mark not exists for 13-th check, retry later table multi_capture_1.usertable exists table multi_capture_2.usertable not exists for 1-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl --cacert /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client.pem --key /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client-key.pem --user ticdc:ticdc_secret -vsL --max-time 20 https://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Initializing NSS with certpath: sql:/etc/pki/nssdb * CAfile: /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem CApath: none * NSS: client certificate from file * subject: CN=client * start date: Feb 18 07:48:00 2020 GMT * expire date: Jan 25 07:48:00 2120 GMT * common name: client * issuer: CN=My own CA,O=PingCAP,L=Beijing,ST=Beijing,C=CN * SSL connection using TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 * Server certificate: * subject: CN=tidb-server * start date: Feb 18 09:11:00 2020 GMT * expire date: Jan 25 09:11:00 2120 GMT * common name: tidb-server * issuer: CN=My own CA,O=PingCAP,L=Beijing,ST=Beijing,C=CN * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 06 Apr 2025 00:20:51 GMT < Content-Length: 1253 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6bb0fc14-b211-4e64-809e-6e5b8f7f2180 {"id":"6bb0fc14-b211-4e64-809e-6e5b8f7f2180","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898849} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/1f5d96087892bfdb 6bb0fc14-b211-4e64-809e-6e5b8f7f2180 /tidb/cdc/default/default/upstream/7489988500234899460 {"id":7489988500234899460,"pd-endpoints":"https://127.0.0.1:2579,https://127.0.0.1:2579","key-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/server-key.pem","cert-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/server.pem","ca-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem","cert-allowed-cn":["client","tidb-server"]}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6bb0fc14-b211-4e64-809e-6e5b8f7f2180 {"id":"6bb0fc14-b211-4e64-809e-6e5b8f7f2180","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898849} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/1f5d96087892bfdb 6bb0fc14-b211-4e64-809e-6e5b8f7f2180 /tidb/cdc/default/default/upstream/7489988500234899460 {"id":7489988500234899460,"pd-endpoints":"https://127.0.0.1:2579,https://127.0.0.1:2579","key-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/server-key.pem","cert-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/server.pem","ca-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem","cert-allowed-cn":["client","tidb-server"]}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6bb0fc14-b211-4e64-809e-6e5b8f7f2180 {"id":"6bb0fc14-b211-4e64-809e-6e5b8f7f2180","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898849} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/1f5d96087892bfdb 6bb0fc14-b211-4e64-809e-6e5b8f7f2180 /tidb/cdc/default/default/upstream/7489988500234899460 {"id":7489988500234899460,"pd-endpoints":"https://127.0.0.1:2579,https://127.0.0.1:2579","key-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/server-key.pem","cert-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/server.pem","ca-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem","cert-allowed-cn":["client","tidb-server"]}' + grep -q 'etcd info' + break + set +x table ddl_manager.finish_mark not exists for 14-th check, retry later Cache restored successfully (ws/jenkins-pingcap-tiflow-periodics_cdc_integration_mysql_test-234/tiflow-cdc) 4028702720 bytes in 11.11 secs (362691964 bytes/sec) [Pipeline] { [Pipeline] cache check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/capture' 'capture' + key_prefix=/tidb/cdc/default/__cdc_meta__/capture + message=capture ++ etcdctl get /tidb/cdc/default/__cdc_meta__/capture --prefix --keys-only + info=/tidb/cdc/default/__cdc_meta__/capture/fa9b673a-72d5-464a-816e-71a04753df77 + [[ /tidb/cdc/default/__cdc_meta__/capture/fa9b673a-72d5-464a-816e-71a04753df77 =~ capture ]] + echo 'capture contains in etcd /tidb/cdc/default/__cdc_meta__/capture/fa9b673a-72d5-464a-816e-71a04753df77' capture contains in etcd /tidb/cdc/default/__cdc_meta__/capture/fa9b673a-72d5-464a-816e-71a04753df77 + echo 'check failed' check failed + exit 1 run task failed 3-th time, retry later table multi_capture_2.usertable exists table multi_capture_3.usertable not exists for 1-th check, retry later table ddl_manager.finish_mark not exists for 15-th check, retry later [Sun Apr 6 08:20:53 CST 2025] <<<<<< START cdc server in http_api_tls case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates ']' + curl_status_cmd='curl --cacert /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client.pem --key /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client-key.pem --user ticdc:ticdc_secret -vsL --max-time 20 https://127.0.0.1:8301/debug/info' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.http_api_tls.61836185.out server --log-file /tmp/tidb_cdc_test/http_api_tls/cdc_http_api_tls_tls2.log --log-level debug --data-dir /tmp/tidb_cdc_test/http_api_tls/cdc_data_http_api_tls_tls2 --cluster-id default --config /tmp/tidb_cdc_test/http_api_tls/server.toml --ca /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/server.pem --key /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/server-key.pem --cert-allowed-cn client --addr 127.0.0.1:8301 --pd https://127.0.0.1:2579 ++ curl --cacert /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client.pem --key /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client-key.pem --user ticdc:ticdc_secret -vsL --max-time 20 https://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table multi_capture_3.usertable exists table multi_capture_4.usertable not exists for 1-th check, retry later table partition_table.t exists table partition_table.t1 exists table partition_table.t2 not exists for 1-th check, retry later table ddl_manager.finish_mark not exists for 16-th check, retry later table partition_table.t2 not exists for 2-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl --cacert /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client.pem --key /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client-key.pem --user ticdc:ticdc_secret -vsL --max-time 20 https://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Initializing NSS with certpath: sql:/etc/pki/nssdb * CAfile: /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem CApath: none * NSS: client certificate from file * subject: CN=client * start date: Feb 18 07:48:00 2020 GMT * expire date: Jan 25 07:48:00 2120 GMT * common name: client * issuer: CN=My own CA,O=PingCAP,L=Beijing,ST=Beijing,C=CN * SSL connection using TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 * Server certificate: * subject: CN=tidb-server * start date: Feb 18 09:11:00 2020 GMT * expire date: Jan 25 09:11:00 2120 GMT * common name: tidb-server * issuer: CN=My own CA,O=PingCAP,L=Beijing,ST=Beijing,C=CN * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 06 Apr 2025 00:20:57 GMT < Content-Length: 1714 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1b9dcede-b441-42cd-82f4-b19826086539 {"id":"1b9dcede-b441-42cd-82f4-b19826086539","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898854} /tidb/cdc/default/__cdc_meta__/capture/6bb0fc14-b211-4e64-809e-6e5b8f7f2180 {"id":"6bb0fc14-b211-4e64-809e-6e5b8f7f2180","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898849} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/1f5d96087892bfdb 6bb0fc14-b211-4e64-809e-6e5b8f7f2180 /tidb/cdc/default/__cdc_meta__/owner/1f5d96087892c019 1b9dcede-b441-42cd-82f4-b19826086539 /tidb/cdc/default/default/upstream/7489988500234899460 {"id":7489988500234899460,"pd-endpoints":"https://127.0.0.1:2579,https://127.0.0.1:2579","key-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/server-key.pem","cert-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/server.pem","ca-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem","cert-allowed-cn":["client","tidb-server"]}' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1b9dcede-b441-42cd-82f4-b19826086539 {"id":"1b9dcede-b441-42cd-82f4-b19826086539","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898854} /tidb/cdc/default/__cdc_meta__/capture/6bb0fc14-b211-4e64-809e-6e5b8f7f2180 {"id":"6bb0fc14-b211-4e64-809e-6e5b8f7f2180","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898849} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/1f5d96087892bfdb 6bb0fc14-b211-4e64-809e-6e5b8f7f2180 /tidb/cdc/default/__cdc_meta__/owner/1f5d96087892c019 1b9dcede-b441-42cd-82f4-b19826086539 /tidb/cdc/default/default/upstream/7489988500234899460 {"id":7489988500234899460,"pd-endpoints":"https://127.0.0.1:2579,https://127.0.0.1:2579","key-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/server-key.pem","cert-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/server.pem","ca-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem","cert-allowed-cn":["client","tidb-server"]}' + grep -q 'failed to get info:' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1b9dcede-b441-42cd-82f4-b19826086539 {"id":"1b9dcede-b441-42cd-82f4-b19826086539","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898854} /tidb/cdc/default/__cdc_meta__/capture/6bb0fc14-b211-4e64-809e-6e5b8f7f2180 {"id":"6bb0fc14-b211-4e64-809e-6e5b8f7f2180","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898849} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/1f5d96087892bfdb 6bb0fc14-b211-4e64-809e-6e5b8f7f2180 /tidb/cdc/default/__cdc_meta__/owner/1f5d96087892c019 1b9dcede-b441-42cd-82f4-b19826086539 /tidb/cdc/default/default/upstream/7489988500234899460 {"id":7489988500234899460,"pd-endpoints":"https://127.0.0.1:2579,https://127.0.0.1:2579","key-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/server-key.pem","cert-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/server.pem","ca-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem","cert-allowed-cn":["client","tidb-server"]}' + grep -q 'etcd info' + break + set +x table multi_capture_4.usertable exists check diff failed 1-th time, retry later table ddl_manager.finish_mark not exists for 17-th check, retry later table partition_table.t2 not exists for 3-th check, retry later check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/capture' 'capture' + key_prefix=/tidb/cdc/default/__cdc_meta__/capture + message=capture ++ etcdctl get /tidb/cdc/default/__cdc_meta__/capture --prefix --keys-only + info= + [[ '' =~ capture ]] + echo 'check pass' check pass + exit 0 run task successfully check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/owner' 'owner' + key_prefix=/tidb/cdc/default/__cdc_meta__/owner + message=owner ++ etcdctl get /tidb/cdc/default/__cdc_meta__/owner --prefix --keys-only + info= + [[ '' =~ owner ]] + echo 'check pass' check pass + exit 0 run task successfully [Sun Apr 6 08:20:58 CST 2025] <<<<<< START cdc server in changefeed_error case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/NewChangefeedRetryError=return(true)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.58345836.out server --log-file /tmp/tidb_cdc_test/changefeed_error/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_error/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 pass test: check health pass test: get status check diff successfully ***************** properties ***************** "readallfields"="true" "recordcount"="20" "dotransactions"="false" "mysql.db"="multi_capture_1" "mysql.port"="4000" "scanproportion"="0" "mysql.user"="root" "requestdistribution"="uniform" "insertproportion"="0" "operationcount"="0" "readproportion"="0" "mysql.host"="127.0.0.1" "updateproportion"="0" "threadcount"="2" "workload"="core" ********************************************** Run finished, takes 17.45408ms INSERT - Takes(s): 0.0, Count: 20, OPS: 1271.8, Avg(us): 1697, Min(us): 578, Max(us): 5260, 95th(us): 6000, 99th(us): 6000 ***************** properties ***************** "requestdistribution"="uniform" "insertproportion"="0" "operationcount"="0" "readallfields"="true" "readproportion"="0" "mysql.host"="127.0.0.1" "scanproportion"="0" "workload"="core" "mysql.db"="multi_capture_2" "mysql.port"="4000" "recordcount"="20" "updateproportion"="0" "dotransactions"="false" "threadcount"="2" "mysql.user"="root" ********************************************** Run finished, takes 15.833957ms INSERT - Takes(s): 0.0, Count: 20, OPS: 1420.8, Avg(us): 1519, Min(us): 571, Max(us): 3585, 95th(us): 4000, 99th(us): 4000 ***************** properties ***************** "readproportion"="0" "operationcount"="0" "mysql.host"="127.0.0.1" "updateproportion"="0" "workload"="core" "dotransactions"="false" "scanproportion"="0" "mysql.user"="root" "readallfields"="true" "recordcount"="20" "mysql.db"="multi_capture_3" "requestdistribution"="uniform" "mysql.port"="4000" "threadcount"="2" "insertproportion"="0" ********************************************** Run finished, takes 13.426431ms INSERT - Takes(s): 0.0, Count: 20, OPS: 1702.6, Avg(us): 1277, Min(us): 570, Max(us): 2520, 95th(us): 2000, 99th(us): 3000 ***************** properties ***************** "threadcount"="2" "mysql.user"="root" "mysql.db"="multi_capture_4" "updateproportion"="0" "mysql.host"="127.0.0.1" "readallfields"="true" "insertproportion"="0" "dotransactions"="false" "requestdistribution"="uniform" "workload"="core" "scanproportion"="0" "recordcount"="20" "readproportion"="0" "operationcount"="0" "mysql.port"="4000" ********************************************** Run finished, takes 14.445886ms INSERT - Takes(s): 0.0, Count: 20, OPS: 1550.3, Avg(us): 1395, Min(us): 677, Max(us): 2169, 95th(us): 3000, 99th(us): 3000 check diff failed 1-th time, retry later table ddl_manager.finish_mark not exists for 18-th check, retry later pass test: create changefeed check_changefeed_state https://127.0.0.1:2579 changefeed-test1 normal null /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates + endpoints=https://127.0.0.1:2579 + changefeed_id=changefeed-test1 + expected_state=normal + error_msg=null + tls_dir=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates + [[ https://127.0.0.1:2579 =~ https ]] ++ cdc cli changefeed query --ca=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client.pem --key=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client-key.pem --pd=https://127.0.0.1:2579 -c changefeed-test1 -s + info='{ "upstream_id": 7489988500234899460, "namespace": "default", "id": "changefeed-test1", "state": "normal", "checkpoint_tso": 457152622639448073, "checkpoint_time": "2025-04-06 08:20:59.556", "error": null }' + echo '{ "upstream_id": 7489988500234899460, "namespace": "default", "id": "changefeed-test1", "state": "normal", "checkpoint_tso": 457152622639448073, "checkpoint_time": "2025-04-06 08:20:59.556", "error": null }' { "upstream_id": 7489988500234899460, "namespace": "default", "id": "changefeed-test1", "state": "normal", "checkpoint_tso": 457152622639448073, "checkpoint_time": "2025-04-06 08:20:59.556", "error": null } ++ echo '{' '"upstream_id":' 7489988500234899460, '"namespace":' '"default",' '"id":' '"changefeed-test1",' '"state":' '"normal",' '"checkpoint_tso":' 457152622639448073, '"checkpoint_time":' '"2025-04-06' '08:20:59.556",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7489988500234899460, '"namespace":' '"default",' '"id":' '"changefeed-test1",' '"state":' '"normal",' '"checkpoint_tso":' 457152622639448073, '"checkpoint_time":' '"2025-04-06' '08:20:59.556",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully check_changefeed_state https://127.0.0.1:2579 changefeed-test2 normal null /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates + endpoints=https://127.0.0.1:2579 + changefeed_id=changefeed-test2 + expected_state=normal + error_msg=null + tls_dir=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates + [[ https://127.0.0.1:2579 =~ https ]] ++ cdc cli changefeed query --ca=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client.pem --key=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client-key.pem --pd=https://127.0.0.1:2579 -c changefeed-test2 -s table partition_table.t2 not exists for 4-th check, retry later + info='{ "upstream_id": 7489988500234899460, "namespace": "default", "id": "changefeed-test2", "state": "normal", "checkpoint_tso": 457152622678769670, "checkpoint_time": "2025-04-06 08:20:59.706", "error": null }' + echo '{ "upstream_id": 7489988500234899460, "namespace": "default", "id": "changefeed-test2", "state": "normal", "checkpoint_tso": 457152622678769670, "checkpoint_time": "2025-04-06 08:20:59.706", "error": null }' { "upstream_id": 7489988500234899460, "namespace": "default", "id": "changefeed-test2", "state": "normal", "checkpoint_tso": 457152622678769670, "checkpoint_time": "2025-04-06 08:20:59.706", "error": null } ++ echo '{' '"upstream_id":' 7489988500234899460, '"namespace":' '"default",' '"id":' '"changefeed-test2",' '"state":' '"normal",' '"checkpoint_tso":' 457152622678769670, '"checkpoint_time":' '"2025-04-06' '08:20:59.706",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7489988500234899460, '"namespace":' '"default",' '"id":' '"changefeed-test2",' '"state":' '"normal",' '"checkpoint_tso":' 457152622678769670, '"checkpoint_time":' '"2025-04-06' '08:20:59.706",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully check_changefeed_state https://127.0.0.1:2579 changefeed-test3 normal null /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates + endpoints=https://127.0.0.1:2579 + changefeed_id=changefeed-test3 + expected_state=normal + error_msg=null + tls_dir=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates + [[ https://127.0.0.1:2579 =~ https ]] ++ cdc cli changefeed query --ca=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client.pem --key=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client-key.pem --pd=https://127.0.0.1:2579 -c changefeed-test3 -s + info='{ "upstream_id": 7489988500234899460, "namespace": "default", "id": "changefeed-test3", "state": "normal", "checkpoint_tso": 457152622718353411, "checkpoint_time": "2025-04-06 08:20:59.857", "error": null }' + echo '{ "upstream_id": 7489988500234899460, "namespace": "default", "id": "changefeed-test3", "state": "normal", "checkpoint_tso": 457152622718353411, "checkpoint_time": "2025-04-06 08:20:59.857", "error": null }' { "upstream_id": 7489988500234899460, "namespace": "default", "id": "changefeed-test3", "state": "normal", "checkpoint_tso": 457152622718353411, "checkpoint_time": "2025-04-06 08:20:59.857", "error": null } ++ echo '{' '"upstream_id":' 7489988500234899460, '"namespace":' '"default",' '"id":' '"changefeed-test3",' '"state":' '"normal",' '"checkpoint_tso":' 457152622718353411, '"checkpoint_time":' '"2025-04-06' '08:20:59.857",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7489988500234899460, '"namespace":' '"default",' '"id":' '"changefeed-test3",' '"state":' '"normal",' '"checkpoint_tso":' 457152622718353411, '"checkpoint_time":' '"2025-04-06' '08:20:59.857",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully check_changefeed_state https://127.0.0.1:2579 changefeed-test4 normal null /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates + endpoints=https://127.0.0.1:2579 + changefeed_id=changefeed-test4 + expected_state=normal + error_msg=null + tls_dir=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates + [[ https://127.0.0.1:2579 =~ https ]] ++ cdc cli changefeed query --ca=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client.pem --key=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client-key.pem --pd=https://127.0.0.1:2579 -c changefeed-test4 -s + info='{ "upstream_id": 7489988500234899460, "namespace": "default", "id": "changefeed-test4", "state": "normal", "checkpoint_tso": 457152622757412868, "checkpoint_time": "2025-04-06 08:21:00.006", "error": null }' + echo '{ "upstream_id": 7489988500234899460, "namespace": "default", "id": "changefeed-test4", "state": "normal", "checkpoint_tso": 457152622757412868, "checkpoint_time": "2025-04-06 08:21:00.006", "error": null }' { "upstream_id": 7489988500234899460, "namespace": "default", "id": "changefeed-test4", "state": "normal", "checkpoint_tso": 457152622757412868, "checkpoint_time": "2025-04-06 08:21:00.006", "error": null } ++ echo '{' '"upstream_id":' 7489988500234899460, '"namespace":' '"default",' '"id":' '"changefeed-test4",' '"state":' '"normal",' '"checkpoint_tso":' 457152622757412868, '"checkpoint_time":' '"2025-04-06' '08:21:00.006",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7489988500234899460, '"namespace":' '"default",' '"id":' '"changefeed-test4",' '"state":' '"normal",' '"checkpoint_tso":' 457152622757412868, '"checkpoint_time":' '"2025-04-06' '08:21:00.006",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully check diff successfully table ddl_manager.finish_mark not exists for 19-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 06 Apr 2025 00:21:02 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/bd77d2fd-6b26-4970-aa12-77be99973db7 {"id":"bd77d2fd-6b26-4970-aa12-77be99973db7","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898859} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 3 /tidb/cdc/default/__cdc_meta__/owner/223196087848f0e1 bd77d2fd-6b26-4970-aa12-77be99973db7 /tidb/cdc/default/default/changefeed/info/changefeed-error {"upstream-id":7489988418452591948,"namespace":"default","changefeed-id":"changefeed-error","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2025-04-06T08:20:40.75154244+08:00","start-ts":457152616123858945,"target-ts":0,"admin-job-type":1,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"pending","error":{"time":"2025-04-06T08:20:59.601991633+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrOwnerUnknown","message":"failpoint injected retriable error"},"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457152622651506689} /tidb/cdc/default/default/changefeed/status/changefeed-error {"checkpoint-ts":457152619099455500,"min-table-barrier-ts":457152619099455500,"admin-job-type":1} /tidb/cdc/default/default/upstream/7489988418452591948 {"id":7489988418452591948,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/bd77d2fd-6b26-4970-aa12-77be99973db7 {"id":"bd77d2fd-6b26-4970-aa12-77be99973db7","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898859} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 3 /tidb/cdc/default/__cdc_meta__/owner/223196087848f0e1 bd77d2fd-6b26-4970-aa12-77be99973db7 /tidb/cdc/default/default/changefeed/info/changefeed-error {"upstream-id":7489988418452591948,"namespace":"default","changefeed-id":"changefeed-error","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2025-04-06T08:20:40.75154244+08:00","start-ts":457152616123858945,"target-ts":0,"admin-job-type":1,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"pending","error":{"time":"2025-04-06T08:20:59.601991633+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrOwnerUnknown","message":"failpoint injected retriable error"},"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457152622651506689} /tidb/cdc/default/default/changefeed/status/changefeed-error {"checkpoint-ts":457152619099455500,"min-table-barrier-ts":457152619099455500,"admin-job-type":1} /tidb/cdc/default/default/upstream/7489988418452591948 {"id":7489988418452591948,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/bd77d2fd-6b26-4970-aa12-77be99973db7 {"id":"bd77d2fd-6b26-4970-aa12-77be99973db7","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898859} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 3 /tidb/cdc/default/__cdc_meta__/owner/223196087848f0e1 bd77d2fd-6b26-4970-aa12-77be99973db7 /tidb/cdc/default/default/changefeed/info/changefeed-error {"upstream-id":7489988418452591948,"namespace":"default","changefeed-id":"changefeed-error","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2025-04-06T08:20:40.75154244+08:00","start-ts":457152616123858945,"target-ts":0,"admin-job-type":1,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"pending","error":{"time":"2025-04-06T08:20:59.601991633+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrOwnerUnknown","message":"failpoint injected retriable error"},"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457152622651506689} /tidb/cdc/default/default/changefeed/status/changefeed-error {"checkpoint-ts":457152619099455500,"min-table-barrier-ts":457152619099455500,"admin-job-type":1} /tidb/cdc/default/default/upstream/7489988418452591948 {"id":7489988418452591948,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x check_changefeed_state http://127.0.0.1:2379 changefeed-error warning failpoint injected retriable error + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-error + expected_state=warning + error_msg=failpoint + tls_dir=error + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-error -s wait process cdc.test exit for 1-th time... + info='{ "upstream_id": 7489988418452591948, "namespace": "default", "id": "changefeed-error", "state": "warning", "checkpoint_tso": 457152619099455500, "checkpoint_time": "2025-04-06 08:20:46.052", "error": { "time": "2025-04-06T08:20:59.601991633+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrOwnerUnknown", "message": "failpoint injected retriable error" } }' + echo '{ "upstream_id": 7489988418452591948, "namespace": "default", "id": "changefeed-error", "state": "warning", "checkpoint_tso": 457152619099455500, "checkpoint_time": "2025-04-06 08:20:46.052", "error": { "time": "2025-04-06T08:20:59.601991633+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrOwnerUnknown", "message": "failpoint injected retriable error" } }' { "upstream_id": 7489988418452591948, "namespace": "default", "id": "changefeed-error", "state": "warning", "checkpoint_tso": 457152619099455500, "checkpoint_time": "2025-04-06 08:20:46.052", "error": { "time": "2025-04-06T08:20:59.601991633+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrOwnerUnknown", "message": "failpoint injected retriable error" } } ++ echo '{' '"upstream_id":' 7489988418452591948, '"namespace":' '"default",' '"id":' '"changefeed-error",' '"state":' '"warning",' '"checkpoint_tso":' 457152619099455500, '"checkpoint_time":' '"2025-04-06' '08:20:46.052",' '"error":' '{' '"time":' '"2025-04-06T08:20:59.601991633+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrOwnerUnknown",' '"message":' '"failpoint' injected retriable 'error"' '}' '}' ++ jq -r .state + state=warning + [[ ! warning == \w\a\r\n\i\n\g ]] ++ echo '{' '"upstream_id":' 7489988418452591948, '"namespace":' '"default",' '"id":' '"changefeed-error",' '"state":' '"warning",' '"checkpoint_tso":' 457152619099455500, '"checkpoint_time":' '"2025-04-06' '08:20:46.052",' '"error":' '{' '"time":' '"2025-04-06T08:20:59.601991633+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrOwnerUnknown",' '"message":' '"failpoint' injected retriable 'error"' '}' '}' ++ jq -r .error.message + message='failpoint injected retriable error' + [[ ! failpoint injected retriable error =~ failpoint ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.5928.out cli changefeed remove -c changefeed-error wait process cdc.test exit for 2-th time... table partition_table.t2 not exists for 5-th check, retry later Changefeed remove successfully. ID: changefeed-error CheckpointTs: 457152619099455500 SinkURI: mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... wait process cdc.test exit for 3-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Sun Apr 6 08:21:03 CST 2025] <<<<<< run test case multi_capture success! >>>>>> table ddl_manager.finish_mark not exists for 20-th check, retry later table test.verify_table_eligible not exists for 1-th check, retry later + set +x check_no_changefeed 127.0.0.1:2379 parse error: Invalid numeric literal at line 1, column 6 run task successfully Cache restored successfully (ws/jenkins-pingcap-tiflow-periodics_cdc_integration_mysql_test-234/tiflow-cdc) 4028702720 bytes in 9.08 secs (443688872 bytes/sec) [Pipeline] { [Pipeline] timeout Timeout set to expire in 45 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 45 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 45 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 45 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 45 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 45 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 45 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 45 min [Pipeline] { table partition_table.t2 exists table partition_table.finish_mark not exists for 1-th check, retry later [Pipeline] timeout Timeout set to expire in 45 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 45 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 45 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 45 min [Pipeline] { [Pipeline] sh wait process cdc.test exit for 1-th time... [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G14 Run cases: changefeed_finish force_replicate_table PROW_JOB_ID=1ff7fcb7-127a-11f0-bebb-96e16cfa81f4 JENKINS_NODE_COOKIE=a7fc675f-a1a1-4af5-a92e-90ba5dd5aa78 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/ GOLANG_VERSION=1.23.3 HOSTNAME=pingcap-tiflow-periodics-cdc-integration-mysql-test-234-m-66zpf HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-periodics_cdc_integration_mysql_test-234 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 JOB_SPEC={"type":"periodic","job":"pingcap/tiflow/periodics_cdc_integration_mysql_test","buildid":"1908671202193313792","prowjobid":"1ff7fcb7-127a-11f0-bebb-96e16cfa81f4","extra_refs":[{"org":"pingcap","repo":"tiflow","base_ref":"master"}]} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ JOB_NAME=pingcap/tiflow/periodics_cdc_integration_mysql_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#234 TEST_GROUP=G14 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1908671202193313792 GOLANG_DOWNLOAD_SHA256=a0afb9744c00648bafb1b90b4aba5bdb86f424f02f9275399ce0c20b93a2c3a8 JOB_BASE_NAME=periodics_cdc_integration_mysql_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_periodics_cdc_integration_mysql_test_234-mqwq4 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-8f9954e717ae06be5a19c339deaa805ddd7d685252d85c8682580d893f46201d NODE_LABELS=pingcap-tiflow-periodics-cdc-integration-mysql-test-234-m-66zpf pingcap_tiflow_periodics_cdc_integration_mysql_test_234-mqwq4 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-periodics-cdc-integration-mysql-test-234-m-66zpf GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/display/redirect BUILD_NUMBER=234 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.23.3.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/changefeed_finish/run.sh using Sink-Type: mysql... <<================= [Pipeline] sh wait process cdc.test exit for 2-th time... table ddl_manager.finish_mark not exists for 21-th check, retry later table test.verify_table_eligible not exists for 2-th check, retry later The 1 times to try to start tidb cluster... [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G10 cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/owner' 'owner' + key_prefix=/tidb/cdc/default/__cdc_meta__/owner + message=owner ++ etcdctl get /tidb/cdc/default/__cdc_meta__/owner --prefix --keys-only + info= + [[ '' =~ owner ]] + echo 'check pass' check pass + exit 0 run task successfully [Sun Apr 6 08:21:06 CST 2025] <<<<<< START cdc server in changefeed_error case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/InjectChangefeedDDLError=return(true)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.60336035.out server --log-file /tmp/tidb_cdc_test/changefeed_error/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_error/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G12 Run cases: default_value simple cdc_server_tips event_filter sql_mode PROW_JOB_ID=1ff7fcb7-127a-11f0-bebb-96e16cfa81f4 JENKINS_NODE_COOKIE=b9b0631e-ab74-45b5-967d-1e5f91dce17e BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/ GOLANG_VERSION=1.23.3 HOSTNAME=pingcap-tiflow-periodics-cdc-integration-mysql-test-234-j-shst4 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-periodics_cdc_integration_mysql_test-234 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 JOB_SPEC={"type":"periodic","job":"pingcap/tiflow/periodics_cdc_integration_mysql_test","buildid":"1908671202193313792","prowjobid":"1ff7fcb7-127a-11f0-bebb-96e16cfa81f4","extra_refs":[{"org":"pingcap","repo":"tiflow","base_ref":"master"}]} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ JOB_NAME=pingcap/tiflow/periodics_cdc_integration_mysql_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#234 TEST_GROUP=G10 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1908671202193313792 GOLANG_DOWNLOAD_SHA256=a0afb9744c00648bafb1b90b4aba5bdb86f424f02f9275399ce0c20b93a2c3a8 JOB_BASE_NAME=periodics_cdc_integration_mysql_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_periodics_cdc_integration_mysql_test_234-jh1tm GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-8f9954e717ae06be5a19c339deaa805ddd7d685252d85c8682580d893f46201d NODE_LABELS=pingcap-tiflow-periodics-cdc-integration-mysql-test-234-j-shst4 pingcap_tiflow_periodics_cdc_integration_mysql_test_234-jh1tm GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-periodics-cdc-integration-mysql-test-234-j-shst4 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/display/redirect BUILD_NUMBER=234 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.23.3.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/default_value/run.sh using Sink-Type: mysql... <<================= find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/simple/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [Pipeline] sh Run cases: many_pk_or_uk capture_session_done_during_task ddl_attributes PROW_JOB_ID=1ff7fcb7-127a-11f0-bebb-96e16cfa81f4 JENKINS_NODE_COOKIE=2650a09b-8e2b-4ef9-b5bf-dbb8a7b0d526 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/ GOLANG_VERSION=1.23.3 HOSTNAME=pingcap-tiflow-periodics-cdc-integration-mysql-test-234-4-tjmd8 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-periodics_cdc_integration_mysql_test-234 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 JOB_SPEC={"type":"periodic","job":"pingcap/tiflow/periodics_cdc_integration_mysql_test","buildid":"1908671202193313792","prowjobid":"1ff7fcb7-127a-11f0-bebb-96e16cfa81f4","extra_refs":[{"org":"pingcap","repo":"tiflow","base_ref":"master"}]} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ JOB_NAME=pingcap/tiflow/periodics_cdc_integration_mysql_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#234 TEST_GROUP=G12 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1908671202193313792 GOLANG_DOWNLOAD_SHA256=a0afb9744c00648bafb1b90b4aba5bdb86f424f02f9275399ce0c20b93a2c3a8 JOB_BASE_NAME=periodics_cdc_integration_mysql_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_periodics_cdc_integration_mysql_test_234-46hg9 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-8f9954e717ae06be5a19c339deaa805ddd7d685252d85c8682580d893f46201d NODE_LABELS=pingcap-tiflow-periodics-cdc-integration-mysql-test-234-4-tjmd8 pingcap_tiflow_periodics_cdc_integration_mysql_test_234-46hg9 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-periodics-cdc-integration-mysql-test-234-4-tjmd8 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/display/redirect BUILD_NUMBER=234 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.23.3.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/many_pk_or_uk/run.sh using Sink-Type: mysql... <<================= + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G18 [Pipeline] sh Run cases: availability http_proxies sequence PROW_JOB_ID=1ff7fcb7-127a-11f0-bebb-96e16cfa81f4 JENKINS_NODE_COOKIE=d13ec1aa-15ae-4e11-8be3-b743a5bd5a01 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/ GOLANG_VERSION=1.23.3 HOSTNAME=pingcap-tiflow-periodics-cdc-integration-mysql-test-234-p-5tjc6 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-periodics_cdc_integration_mysql_test-234 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 JOB_SPEC={"type":"periodic","job":"pingcap/tiflow/periodics_cdc_integration_mysql_test","buildid":"1908671202193313792","prowjobid":"1ff7fcb7-127a-11f0-bebb-96e16cfa81f4","extra_refs":[{"org":"pingcap","repo":"tiflow","base_ref":"master"}]} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ JOB_NAME=pingcap/tiflow/periodics_cdc_integration_mysql_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#234 TEST_GROUP=G18 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1908671202193313792 GOLANG_DOWNLOAD_SHA256=a0afb9744c00648bafb1b90b4aba5bdb86f424f02f9275399ce0c20b93a2c3a8 JOB_BASE_NAME=periodics_cdc_integration_mysql_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_periodics_cdc_integration_mysql_test_234-p2wz0 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-8f9954e717ae06be5a19c339deaa805ddd7d685252d85c8682580d893f46201d NODE_LABELS=pingcap_tiflow_periodics_cdc_integration_mysql_test_234-p2wz0 pingcap-tiflow-periodics-cdc-integration-mysql-test-234-p-5tjc6 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-periodics-cdc-integration-mysql-test-234-p-5tjc6 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/display/redirect BUILD_NUMBER=234 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.23.3.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/availability/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G09 [Pipeline] sh table partition_table.finish_mark not exists for 2-th check, retry later Run cases: gc_safepoint changefeed_pause_resume cli_with_auth savepoint synced_status PROW_JOB_ID=1ff7fcb7-127a-11f0-bebb-96e16cfa81f4 JENKINS_NODE_COOKIE=6f1c0ceb-d2d2-4ed6-8c5a-1c668ad658ab BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/ GOLANG_VERSION=1.23.3 HOSTNAME=pingcap-tiflow-periodics-cdc-integration-mysql-test-234-c-g3td9 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-periodics_cdc_integration_mysql_test-234 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 JOB_SPEC={"type":"periodic","job":"pingcap/tiflow/periodics_cdc_integration_mysql_test","buildid":"1908671202193313792","prowjobid":"1ff7fcb7-127a-11f0-bebb-96e16cfa81f4","extra_refs":[{"org":"pingcap","repo":"tiflow","base_ref":"master"}]} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ JOB_NAME=pingcap/tiflow/periodics_cdc_integration_mysql_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#234 TEST_GROUP=G09 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1908671202193313792 GOLANG_DOWNLOAD_SHA256=a0afb9744c00648bafb1b90b4aba5bdb86f424f02f9275399ce0c20b93a2c3a8 JOB_BASE_NAME=periodics_cdc_integration_mysql_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_periodics_cdc_integration_mysql_test_234-cqxrw GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-8f9954e717ae06be5a19c339deaa805ddd7d685252d85c8682580d893f46201d NODE_LABELS=pingcap_tiflow_periodics_cdc_integration_mysql_test_234-cqxrw pingcap-tiflow-periodics-cdc-integration-mysql-test-234-c-g3td9 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-periodics-cdc-integration-mysql-test-234-c-g3td9 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/display/redirect BUILD_NUMBER=234 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.23.3.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/gc_safepoint/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G11 Run cases: resolve_lock move_table autorandom generate_column PROW_JOB_ID=1ff7fcb7-127a-11f0-bebb-96e16cfa81f4 JENKINS_NODE_COOKIE=49159389-109d-4ce9-8512-4b6f42ac7dd4 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/ GOLANG_VERSION=1.23.3 HOSTNAME=pingcap-tiflow-periodics-cdc-integration-mysql-test-234-j-vgmw9 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-periodics_cdc_integration_mysql_test-234 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 JOB_SPEC={"type":"periodic","job":"pingcap/tiflow/periodics_cdc_integration_mysql_test","buildid":"1908671202193313792","prowjobid":"1ff7fcb7-127a-11f0-bebb-96e16cfa81f4","extra_refs":[{"org":"pingcap","repo":"tiflow","base_ref":"master"}]} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ JOB_NAME=pingcap/tiflow/periodics_cdc_integration_mysql_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#234 TEST_GROUP=G11 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1908671202193313792 GOLANG_DOWNLOAD_SHA256=a0afb9744c00648bafb1b90b4aba5bdb86f424f02f9275399ce0c20b93a2c3a8 JOB_BASE_NAME=periodics_cdc_integration_mysql_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_periodics_cdc_integration_mysql_test_234-jpjd1 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-8f9954e717ae06be5a19c339deaa805ddd7d685252d85c8682580d893f46201d NODE_LABELS=pingcap-tiflow-periodics-cdc-integration-mysql-test-234-j-vgmw9 pingcap_tiflow_periodics_cdc_integration_mysql_test_234-jpjd1 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-periodics-cdc-integration-mysql-test-234-j-vgmw9 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/display/redirect BUILD_NUMBER=234 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.23.3.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory [Pipeline] sh =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/resolve_lock/run.sh using Sink-Type: mysql... <<================= + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G15 Run cases: new_ci_collation batch_add_table multi_rocks ci_collation_compatibility PROW_JOB_ID=1ff7fcb7-127a-11f0-bebb-96e16cfa81f4 JENKINS_NODE_COOKIE=62ae5914-9cb1-45a0-80e2-238e46d13daa BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/ GOLANG_VERSION=1.23.3 HOSTNAME=pingcap-tiflow-periodics-cdc-integration-mysql-test-234-x-hbn5c HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-periodics_cdc_integration_mysql_test-234 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 JOB_SPEC={"type":"periodic","job":"pingcap/tiflow/periodics_cdc_integration_mysql_test","buildid":"1908671202193313792","prowjobid":"1ff7fcb7-127a-11f0-bebb-96e16cfa81f4","extra_refs":[{"org":"pingcap","repo":"tiflow","base_ref":"master"}]} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ JOB_NAME=pingcap/tiflow/periodics_cdc_integration_mysql_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#234 TEST_GROUP=G15 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1908671202193313792 GOLANG_DOWNLOAD_SHA256=a0afb9744c00648bafb1b90b4aba5bdb86f424f02f9275399ce0c20b93a2c3a8 JOB_BASE_NAME=periodics_cdc_integration_mysql_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_periodics_cdc_integration_mysql_test_234-xns4p GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-8f9954e717ae06be5a19c339deaa805ddd7d685252d85c8682580d893f46201d NODE_LABELS=pingcap-tiflow-periodics-cdc-integration-mysql-test-234-x-hbn5c pingcap_tiflow_periodics_cdc_integration_mysql_test_234-xns4p GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-periodics-cdc-integration-mysql-test-234-x-hbn5c GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/display/redirect BUILD_NUMBER=234 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.23.3.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/new_ci_collation/run.sh using Sink-Type: mysql... <<================= [Pipeline] sh The 1 times to try to start tidb cluster... + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G16 Run cases: owner_resign processor_etcd_worker_delay sink_hang PROW_JOB_ID=1ff7fcb7-127a-11f0-bebb-96e16cfa81f4 JENKINS_NODE_COOKIE=7ebe8260-f8df-4b49-bbb2-0912574d9fac BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/ GOLANG_VERSION=1.23.3 HOSTNAME=pingcap-tiflow-periodics-cdc-integration-mysql-test-234-2-dwpgc HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-periodics_cdc_integration_mysql_test-234 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 JOB_SPEC={"type":"periodic","job":"pingcap/tiflow/periodics_cdc_integration_mysql_test","buildid":"1908671202193313792","prowjobid":"1ff7fcb7-127a-11f0-bebb-96e16cfa81f4","extra_refs":[{"org":"pingcap","repo":"tiflow","base_ref":"master"}]} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ JOB_NAME=pingcap/tiflow/periodics_cdc_integration_mysql_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#234 TEST_GROUP=G16 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1908671202193313792 GOLANG_DOWNLOAD_SHA256=a0afb9744c00648bafb1b90b4aba5bdb86f424f02f9275399ce0c20b93a2c3a8 JOB_BASE_NAME=periodics_cdc_integration_mysql_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_periodics_cdc_integration_mysql_test_234-20122 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-8f9954e717ae06be5a19c339deaa805ddd7d685252d85c8682580d893f46201d NODE_LABELS=pingcap-tiflow-periodics-cdc-integration-mysql-test-234-2-dwpgc pingcap_tiflow_periodics_cdc_integration_mysql_test_234-20122 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-periodics-cdc-integration-mysql-test-234-2-dwpgc GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/display/redirect BUILD_NUMBER=234 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.23.3.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/owner_resign/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G21 [Pipeline] sh Run cases: bank kill_owner_with_ddl owner_remove_table_error PROW_JOB_ID=1ff7fcb7-127a-11f0-bebb-96e16cfa81f4 JENKINS_NODE_COOKIE=7d58aaac-e19a-4250-a10b-f1aa2fafe11c BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/ GOLANG_VERSION=1.23.3 HOSTNAME=pingcap-tiflow-periodics-cdc-integration-mysql-test-234-x-pbmg2 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-periodics_cdc_integration_mysql_test-234 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 JOB_SPEC={"type":"periodic","job":"pingcap/tiflow/periodics_cdc_integration_mysql_test","buildid":"1908671202193313792","prowjobid":"1ff7fcb7-127a-11f0-bebb-96e16cfa81f4","extra_refs":[{"org":"pingcap","repo":"tiflow","base_ref":"master"}]} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ JOB_NAME=pingcap/tiflow/periodics_cdc_integration_mysql_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#234 TEST_GROUP=G21 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1908671202193313792 GOLANG_DOWNLOAD_SHA256=a0afb9744c00648bafb1b90b4aba5bdb86f424f02f9275399ce0c20b93a2c3a8 JOB_BASE_NAME=periodics_cdc_integration_mysql_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_periodics_cdc_integration_mysql_test_234-x37n0 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-8f9954e717ae06be5a19c339deaa805ddd7d685252d85c8682580d893f46201d NODE_LABELS=pingcap_tiflow_periodics_cdc_integration_mysql_test_234-x37n0 pingcap-tiflow-periodics-cdc-integration-mysql-test-234-x-pbmg2 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-periodics-cdc-integration-mysql-test-234-x-pbmg2 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/display/redirect BUILD_NUMBER=234 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.23.3.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/bank/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G17 Run cases: clustered_index processor_resolved_ts_fallback PROW_JOB_ID=1ff7fcb7-127a-11f0-bebb-96e16cfa81f4 JENKINS_NODE_COOKIE=88c9901a-e848-4661-bc2c-cd075f3b2c69 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/ GOLANG_VERSION=1.23.3 HOSTNAME=pingcap-tiflow-periodics-cdc-integration-mysql-test-234-j-x7t22 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-periodics_cdc_integration_mysql_test-234 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 JOB_SPEC={"type":"periodic","job":"pingcap/tiflow/periodics_cdc_integration_mysql_test","buildid":"1908671202193313792","prowjobid":"1ff7fcb7-127a-11f0-bebb-96e16cfa81f4","extra_refs":[{"org":"pingcap","repo":"tiflow","base_ref":"master"}]} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ JOB_NAME=pingcap/tiflow/periodics_cdc_integration_mysql_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#234 TEST_GROUP=G17 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1908671202193313792 GOLANG_DOWNLOAD_SHA256=a0afb9744c00648bafb1b90b4aba5bdb86f424f02f9275399ce0c20b93a2c3a8 JOB_BASE_NAME=periodics_cdc_integration_mysql_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_periodics_cdc_integration_mysql_test_234-jn7pm GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-8f9954e717ae06be5a19c339deaa805ddd7d685252d85c8682580d893f46201d NODE_LABELS=pingcap-tiflow-periodics-cdc-integration-mysql-test-234-j-x7t22 pingcap_tiflow_periodics_cdc_integration_mysql_test_234-jn7pm GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-periodics-cdc-integration-mysql-test-234-j-x7t22 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/display/redirect BUILD_NUMBER=234 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.23.3.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/clustered_index/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [Pipeline] sh table ddl_manager.finish_mark not exists for 22-th check, retry later table test.verify_table_eligible exists pass test: list changefeed + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G20 Run cases: tidb_mysql_test ddl_reentrant multi_cdc_cluster PROW_JOB_ID=1ff7fcb7-127a-11f0-bebb-96e16cfa81f4 JENKINS_NODE_COOKIE=bd88a97e-e165-49d7-82f2-976d0e463105 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/ GOLANG_VERSION=1.23.3 HOSTNAME=pingcap-tiflow-periodics-cdc-integration-mysql-test-234-z-4sl2f HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-periodics_cdc_integration_mysql_test-234 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 JOB_SPEC={"type":"periodic","job":"pingcap/tiflow/periodics_cdc_integration_mysql_test","buildid":"1908671202193313792","prowjobid":"1ff7fcb7-127a-11f0-bebb-96e16cfa81f4","extra_refs":[{"org":"pingcap","repo":"tiflow","base_ref":"master"}]} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ JOB_NAME=pingcap/tiflow/periodics_cdc_integration_mysql_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#234 TEST_GROUP=G20 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1908671202193313792 GOLANG_DOWNLOAD_SHA256=a0afb9744c00648bafb1b90b4aba5bdb86f424f02f9275399ce0c20b93a2c3a8 JOB_BASE_NAME=periodics_cdc_integration_mysql_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_periodics_cdc_integration_mysql_test_234-zrbwf GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-8f9954e717ae06be5a19c339deaa805ddd7d685252d85c8682580d893f46201d NODE_LABELS=pingcap_tiflow_periodics_cdc_integration_mysql_test_234-zrbwf pingcap-tiflow-periodics-cdc-integration-mysql-test-234-z-4sl2f GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-periodics-cdc-integration-mysql-test-234-z-4sl2f GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/display/redirect BUILD_NUMBER=234 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.23.3.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/tidb_mysql_test/run.sh using Sink-Type: mysql... <<================= + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G13 Run cases: tiflash region_merge common_1 PROW_JOB_ID=1ff7fcb7-127a-11f0-bebb-96e16cfa81f4 JENKINS_NODE_COOKIE=b5830d54-0d65-4f2d-a3af-6e2534e55afb BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/ GOLANG_VERSION=1.23.3 HOSTNAME=pingcap-tiflow-periodics-cdc-integration-mysql-test-234-x-4tfw6 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-periodics_cdc_integration_mysql_test-234 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 JOB_SPEC={"type":"periodic","job":"pingcap/tiflow/periodics_cdc_integration_mysql_test","buildid":"1908671202193313792","prowjobid":"1ff7fcb7-127a-11f0-bebb-96e16cfa81f4","extra_refs":[{"org":"pingcap","repo":"tiflow","base_ref":"master"}]} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ JOB_NAME=pingcap/tiflow/periodics_cdc_integration_mysql_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#234 TEST_GROUP=G13 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1908671202193313792 GOLANG_DOWNLOAD_SHA256=a0afb9744c00648bafb1b90b4aba5bdb86f424f02f9275399ce0c20b93a2c3a8 JOB_BASE_NAME=periodics_cdc_integration_mysql_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_periodics_cdc_integration_mysql_test_234-x0k9t GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-8f9954e717ae06be5a19c339deaa805ddd7d685252d85c8682580d893f46201d NODE_LABELS=pingcap_tiflow_periodics_cdc_integration_mysql_test_234-x0k9t pingcap-tiflow-periodics-cdc-integration-mysql-test-234-x-4tfw6 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-periodics-cdc-integration-mysql-test-234-x-4tfw6 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/display/redirect BUILD_NUMBER=234 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.23.3.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/tiflash/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... table partition_table.finish_mark not exists for 3-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 06 Apr 2025 00:21:09 GMT < Content-Length: 845 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5f84a4f7-127a-42d3-ba19-39e0aca5e9d0 {"id":"5f84a4f7-127a-42d3-ba19-39e0aca5e9d0","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898866} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 4 /tidb/cdc/default/__cdc_meta__/owner/223196087848f144 5f84a4f7-127a-42d3-ba19-39e0aca5e9d0 /tidb/cdc/default/default/upstream/7489988418452591948 {"id":7489988418452591948,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5f84a4f7-127a-42d3-ba19-39e0aca5e9d0 {"id":"5f84a4f7-127a-42d3-ba19-39e0aca5e9d0","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898866} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 4 /tidb/cdc/default/__cdc_meta__/owner/223196087848f144 5f84a4f7-127a-42d3-ba19-39e0aca5e9d0 /tidb/cdc/default/default/upstream/7489988418452591948 {"id":7489988418452591948,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5f84a4f7-127a-42d3-ba19-39e0aca5e9d0 {"id":"5f84a4f7-127a-42d3-ba19-39e0aca5e9d0","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898866} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 4 /tidb/cdc/default/__cdc_meta__/owner/223196087848f144 5f84a4f7-127a-42d3-ba19-39e0aca5e9d0 /tidb/cdc/default/default/upstream/7489988418452591948 {"id":7489988418452591948,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.6087.out cli changefeed create --start-ts=457152616123858945 '--sink-uri=mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1' -c changefeed-error-1 Create changefeed successfully! ID: changefeed-error-1 Info: {"upstream_id":7489988418452591948,"namespace":"default","id":"changefeed-error-1","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create_time":"2025-04-06T08:21:09.566962834+08:00","start_ts":457152616123858945,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457152616123858945,"checkpoint_ts":457152616123858945,"checkpoint_time":"2025-04-06 08:20:34.701"} PASS pass test: get changefeed coverage: 2.4% of statements in github.com/pingcap/tiflow/... start tidb cluster in /tmp/tidb_cdc_test/changefeed_finish Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... table ddl_manager.finish_mark not exists for 23-th check, retry later The 1 times to try to start tidb cluster... The 1 times to try to start tidb cluster... table partition_table.finish_mark not exists for 4-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/new_ci_collation Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... start tidb cluster in /tmp/tidb_cdc_test/owner_resign Starting Upstream PD... + set +x Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/simple Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/clustered_index Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... check_changefeed_status 127.0.0.1:8300 changefeed-error-1 warning last_warning ErrExecDDLFailed + endpoint=127.0.0.1:8300 + changefeed_id=changefeed-error-1 + expected_state=warning + field=last_warning + error_pattern=ErrExecDDLFailed ++ curl 127.0.0.1:8300/api/v2/changefeeds/changefeed-error-1/status % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 252 100 252 0 0 2512 0 --:--:-- --:--:-- --:--:-- 2520 + info='{"state":"warning","resolved_ts":457152616503967756,"checkpoint_ts":457152616503967756,"last_warning":{"time":"2025-04-06T08:21:11.34561071+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrExecDDLFailed","message":"[CDC:ErrExecDDLFailed]exec DDL failed"}}' + echo '{"state":"warning","resolved_ts":457152616503967756,"checkpoint_ts":457152616503967756,"last_warning":{"time":"2025-04-06T08:21:11.34561071+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrExecDDLFailed","message":"[CDC:ErrExecDDLFailed]exec DDL failed"}}' {"state":"warning","resolved_ts":457152616503967756,"checkpoint_ts":457152616503967756,"last_warning":{"time":"2025-04-06T08:21:11.34561071+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrExecDDLFailed","message":"[CDC:ErrExecDDLFailed]exec DDL failed"}} ++ echo '{"state":"warning","resolved_ts":457152616503967756,"checkpoint_ts":457152616503967756,"last_warning":{"time":"2025-04-06T08:21:11.34561071+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrExecDDLFailed","message":"[CDC:ErrExecDDLFailed]exec' DDL 'failed"}}' ++ jq -r .state + state=warning + [[ ! warning == \w\a\r\n\i\n\g ]] + [[ -z last_warning ]] ++ echo '{"state":"warning","resolved_ts":457152616503967756,"checkpoint_ts":457152616503967756,"last_warning":{"time":"2025-04-06T08:21:11.34561071+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrExecDDLFailed","message":"[CDC:ErrExecDDLFailed]exec' DDL 'failed"}}' ++ jq -r .last_warning.message + error_msg='[CDC:ErrExecDDLFailed]exec DDL failed' + [[ ! [CDC:ErrExecDDLFailed]exec DDL failed =~ ErrExecDDLFailed ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.6138.out cli changefeed remove -c changefeed-error-1 start tidb cluster in /tmp/tidb_cdc_test/availability Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... table ddl_manager.finish_mark not exists for 24-th check, retry later pass test: pause changefeed start tidb cluster in /tmp/tidb_cdc_test/gc_safepoint Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... Changefeed remove successfully. ID: changefeed-error-1 CheckpointTs: 457152616503967756 SinkURI: mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... table partition_table.finish_mark not exists for 5-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release The 1 times to try to start tidb cluster... Verifying downstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/tiflash Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/bank Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... pass test: update changefeed Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/synced_status_with_redo/run.sh using Sink-Type: mysql... <<================= +++ dirname /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/synced_status_with_redo/run.sh ++ cd /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/synced_status_with_redo ++ pwd + CUR=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/synced_status_with_redo + source /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/synced_status_with_redo/../_utils/test_prepare ++ UP_TIDB_HOST=127.0.0.1 ++ UP_TIDB_PORT=4000 ++ UP_TIDB_OTHER_PORT=4001 ++ UP_TIDB_STATUS=10080 ++ UP_TIDB_OTHER_STATUS=10081 ++ DOWN_TIDB_HOST=127.0.0.1 ++ DOWN_TIDB_PORT=3306 ++ DOWN_TIDB_STATUS=20080 ++ TLS_TIDB_HOST=127.0.0.1 ++ TLS_TIDB_PORT=3307 ++ TLS_TIDB_STATUS=30080 ++ UP_PD_HOST_1=127.0.0.1 ++ UP_PD_PORT_1=2379 ++ UP_PD_PEER_PORT_1=2380 ++ UP_PD_HOST_2=127.0.0.1 ++ UP_PD_PORT_2=2679 ++ UP_PD_PEER_PORT_2=2680 ++ UP_PD_HOST_3=127.0.0.1 ++ UP_PD_PORT_3=2779 ++ UP_PD_PEER_PORT_3=2780 ++ DOWN_PD_HOST=127.0.0.1 ++ DOWN_PD_PORT=2479 ++ DOWN_PD_PEER_PORT=2480 ++ TLS_PD_HOST=127.0.0.1 ++ TLS_PD_PORT=2579 ++ TLS_PD_PEER_PORT=2580 ++ UP_TIKV_HOST_1=127.0.0.1 ++ UP_TIKV_PORT_1=20160 ++ UP_TIKV_STATUS_PORT_1=20181 ++ UP_TIKV_HOST_2=127.0.0.1 ++ UP_TIKV_PORT_2=20161 ++ UP_TIKV_STATUS_PORT_2=20182 ++ UP_TIKV_HOST_3=127.0.0.1 ++ UP_TIKV_PORT_3=20162 ++ UP_TIKV_STATUS_PORT_3=20183 ++ DOWN_TIKV_HOST=127.0.0.1 ++ DOWN_TIKV_PORT=21160 ++ DOWN_TIKV_STATUS_PORT=21180 ++ TLS_TIKV_HOST=127.0.0.1 ++ TLS_TIKV_PORT=22160 ++ TLS_TIKV_STATUS_PORT=22180 +++ cat /tmp/tidb_cdc_test/KAFKA_VERSION +++ echo 2.4.1 ++ KAFKA_VERSION=2.4.1 + WORK_DIR=/tmp/tidb_cdc_test/synced_status_with_redo + CDC_BINARY=cdc.test + SINK_TYPE=mysql + CDC_COUNT=3 + DB_COUNT=4 + trap stop_tidb_cluster EXIT + run_normal_case_and_unavailable_pd conf/changefeed-redo.toml + rm -rf /tmp/tidb_cdc_test/synced_status_with_redo + mkdir -p /tmp/tidb_cdc_test/synced_status_with_redo + start_tidb_cluster --workdir /tmp/tidb_cdc_test/synced_status_with_redo The 1 times to try to start tidb cluster... + set +x table ddl_manager.finish_mark not exists for 25-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release wait process cdc.test exit for 1-th time... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release start tidb cluster in /tmp/tidb_cdc_test/resolve_lock Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... wait process cdc.test exit for 2-th time... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release pass test: resume changefeed table partition_table.finish_mark not exists for 6-th check, retry later Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/owner' 'owner' + key_prefix=/tidb/cdc/default/__cdc_meta__/owner + message=owner ++ etcdctl get /tidb/cdc/default/__cdc_meta__/owner --prefix --keys-only + info= + [[ '' =~ owner ]] + echo 'check pass' check pass + exit 0 run task successfully [Sun Apr 6 08:21:15 CST 2025] <<<<<< START cdc server in changefeed_error case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/tiflow/pkg/txnutil/gc/InjectActualGCSafePoint=return(9223372036854775807)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.62066208.out server --log-file /tmp/tidb_cdc_test/changefeed_error/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_error/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release start tidb cluster in /tmp/tidb_cdc_test/many_pk_or_uk Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... table ddl_manager.finish_mark not exists for 26-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release start tidb cluster in /tmp/tidb_cdc_test/tidb_mysql_test Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... [Pipeline] withEnv [Pipeline] { Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) pass test: rebalance table Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... table partition_table.finish_mark not exists for 7-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/synced_status_with_redo Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... pass test: list processors Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 27-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 06 Apr 2025 00:21:18 GMT < Content-Length: 845 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c4d8c184-4ec9-440f-95d0-de8ded351ddb {"id":"c4d8c184-4ec9-440f-95d0-de8ded351ddb","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898875} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 5 /tidb/cdc/default/__cdc_meta__/owner/223196087848f1be c4d8c184-4ec9-440f-95d0-de8ded351ddb /tidb/cdc/default/default/upstream/7489988418452591948 {"id":7489988418452591948,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c4d8c184-4ec9-440f-95d0-de8ded351ddb {"id":"c4d8c184-4ec9-440f-95d0-de8ded351ddb","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898875} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 5 /tidb/cdc/default/__cdc_meta__/owner/223196087848f1be c4d8c184-4ec9-440f-95d0-de8ded351ddb /tidb/cdc/default/default/upstream/7489988418452591948 {"id":7489988418452591948,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c4d8c184-4ec9-440f-95d0-de8ded351ddb {"id":"c4d8c184-4ec9-440f-95d0-de8ded351ddb","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898875} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 5 /tidb/cdc/default/__cdc_meta__/owner/223196087848f1be c4d8c184-4ec9-440f-95d0-de8ded351ddb /tidb/cdc/default/default/upstream/7489988418452591948 {"id":7489988418452591948,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.6260.out cli changefeed create --start-ts=457152616123858945 '--sink-uri=mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1' -c changefeed-error-2 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Create changefeed successfully! ID: changefeed-error-2 Info: {"upstream_id":7489988418452591948,"namespace":"default","id":"changefeed-error-2","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create_time":"2025-04-06T08:21:18.865243282+08:00","start_ts":457152616123858945,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457152616123858945,"checkpoint_ts":457152616123858945,"checkpoint_time":"2025-04-06 08:20:34.701"} PASS VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489988621899277058 TiDB Cluster ID. tikv_gc_leader_uuid 65821e46edc000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-m-66zpf, pid:1224, start at 2025-04-06 08:21:16.870967124 +0800 CST m=+2.117396912 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:23:16.880 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:21:16.905 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:11:16.905 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489988621899277058 TiDB Cluster ID. tikv_gc_leader_uuid 65821e46edc000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-m-66zpf, pid:1224, start at 2025-04-06 08:21:16.870967124 +0800 CST m=+2.117396912 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:23:16.880 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:21:16.905 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:11:16.905 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489988620106659806 TiDB Cluster ID. tikv_gc_leader_uuid 65821e47074000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-m-66zpf, pid:1291, start at 2025-04-06 08:21:16.973110976 +0800 CST m=+2.142845076 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:23:16.984 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:21:16.957 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:11:16.957 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table partition_table.finish_mark not exists for 8-th check, retry later Verifying downstream PD is started... coverage: 2.4% of statements in github.com/pingcap/tiflow/... Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/changefeed_finish/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/changefeed_finish/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/changefeed_finish/tiflash/db/proxy"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/changefeed_finish/tiflash-proxy.toml"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/changefeed_finish/tiflash/log/proxy.log"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [Pipeline] container [Pipeline] { ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489988621880289090 TiDB Cluster ID. tikv_gc_leader_uuid 65821e48a840004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-j-shst4, pid:1360, start at 2025-04-06 08:21:18.632119396 +0800 CST m=+2.819284098 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:23:18.642 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:21:18.625 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:11:18.625 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489988621880289090 TiDB Cluster ID. tikv_gc_leader_uuid 65821e48a840004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-j-shst4, pid:1360, start at 2025-04-06 08:21:18.632119396 +0800 CST m=+2.819284098 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:23:18.642 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:21:18.625 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:11:18.625 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489988625747555256 TiDB Cluster ID. tikv_gc_leader_uuid 65821e48ca40004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-j-shst4, pid:1418, start at 2025-04-06 08:21:18.769034616 +0800 CST m=+2.894051151 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:23:18.779 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:21:18.761 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:11:18.761 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 28-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/simple/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/simple/tiflash/log/error.log arg matches is ArgMatches { args: {"config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/simple/tiflash-proxy.toml"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/simple/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/simple/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + set +x check_changefeed_state http://127.0.0.1:2379 changefeed-error-2 failed [CDC:ErrSnapshotLostByGC] + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-error-2 + expected_state=failed + error_msg='[CDC:ErrSnapshotLostByGC]' + tls_dir='[CDC:ErrSnapshotLostByGC]' + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-error-2 -s + info='{ "upstream_id": 7489988418452591948, "namespace": "default", "id": "changefeed-error-2", "state": "failed", "checkpoint_tso": 457152616123858945, "checkpoint_time": "2025-04-06 08:20:34.701", "error": { "time": "2025-04-06T08:21:18.94441976+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrSnapshotLostByGC", "message": "[CDC:ErrSnapshotLostByGC]fail to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 457152616123858945 is earlier than or equal to GC safepoint at 9223372036854775807" } }' + echo '{ "upstream_id": 7489988418452591948, "namespace": "default", "id": "changefeed-error-2", "state": "failed", "checkpoint_tso": 457152616123858945, "checkpoint_time": "2025-04-06 08:20:34.701", "error": { "time": "2025-04-06T08:21:18.94441976+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrSnapshotLostByGC", "message": "[CDC:ErrSnapshotLostByGC]fail to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 457152616123858945 is earlier than or equal to GC safepoint at 9223372036854775807" } }' { "upstream_id": 7489988418452591948, "namespace": "default", "id": "changefeed-error-2", "state": "failed", "checkpoint_tso": 457152616123858945, "checkpoint_time": "2025-04-06 08:20:34.701", "error": { "time": "2025-04-06T08:21:18.94441976+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrSnapshotLostByGC", "message": "[CDC:ErrSnapshotLostByGC]fail to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 457152616123858945 is earlier than or equal to GC safepoint at 9223372036854775807" } } ++ echo '{' '"upstream_id":' 7489988418452591948, '"namespace":' '"default",' '"id":' '"changefeed-error-2",' '"state":' '"failed",' '"checkpoint_tso":' 457152616123858945, '"checkpoint_time":' '"2025-04-06' '08:20:34.701",' '"error":' '{' '"time":' '"2025-04-06T08:21:18.94441976+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrSnapshotLostByGC",' '"message":' '"[CDC:ErrSnapshotLostByGC]fail' to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 457152616123858945 is earlier than or equal to GC safepoint at '9223372036854775807"' '}' '}' ++ jq -r .state VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489988628702016704 TiDB Cluster ID. tikv_gc_leader_uuid 65821e494400007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-x-4tfw6, pid:1342, start at 2025-04-06 08:21:19.270696945 +0800 CST m=+2.762667901 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:23:19.282 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:21:19.248 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:11:19.248 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489988629164751674 TiDB Cluster ID. tikv_gc_leader_uuid 65821e486f40001 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-2-dwpgc, pid:1242, start at 2025-04-06 08:21:18.397431774 +0800 CST m=+2.259386458 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:23:18.405 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:21:18.397 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:11:18.397 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489988629164751674 TiDB Cluster ID. tikv_gc_leader_uuid 65821e486f40001 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-2-dwpgc, pid:1242, start at 2025-04-06 08:21:18.397431774 +0800 CST m=+2.259386458 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:23:18.405 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:21:18.397 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:11:18.397 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489988618596026933 TiDB Cluster ID. tikv_gc_leader_uuid 65821e48a0c000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-2-dwpgc, pid:1299, start at 2025-04-06 08:21:18.613826468 +0800 CST m=+2.398283576 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:23:18.622 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:21:18.595 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:11:18.595 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/owner_resign/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/owner_resign/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/owner_resign/tiflash/db/proxy"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/owner_resign/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/owner_resign/tiflash/log/proxy.log"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + state=failed + [[ ! failed == \f\a\i\l\e\d ]] ++ echo '{' '"upstream_id":' 7489988418452591948, '"namespace":' '"default",' '"id":' '"changefeed-error-2",' '"state":' '"failed",' '"checkpoint_tso":' 457152616123858945, '"checkpoint_time":' '"2025-04-06' '08:20:34.701",' '"error":' '{' '"time":' '"2025-04-06T08:21:18.94441976+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrSnapshotLostByGC",' '"message":' '"[CDC:ErrSnapshotLostByGC]fail' to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 457152616123858945 is earlier than or equal to GC safepoint at '9223372036854775807"' '}' '}' ++ jq -r .error.message + message='[CDC:ErrSnapshotLostByGC]fail to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 457152616123858945 is earlier than or equal to GC safepoint at 9223372036854775807' + [[ ! [CDC:ErrSnapshotLostByGC]fail to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 457152616123858945 is earlier than or equal to GC safepoint at 9223372036854775807 =~ \[CDC:ErrSnapshotLostByGC] ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.6328.out cli changefeed remove -c changefeed-error-2 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489988628380945915 TiDB Cluster ID. tikv_gc_leader_uuid 65821e48cb4000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-p-5tjc6, pid:1369, start at 2025-04-06 08:21:18.782382607 +0800 CST m=+2.564543795 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:23:18.791 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:21:18.826 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:11:18.826 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489988628380945915 TiDB Cluster ID. tikv_gc_leader_uuid 65821e48cb4000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-p-5tjc6, pid:1369, start at 2025-04-06 08:21:18.782382607 +0800 CST m=+2.564543795 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:23:18.791 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:21:18.826 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:11:18.826 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489988628762457630 TiDB Cluster ID. tikv_gc_leader_uuid 65821e48ca40009 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-p-5tjc6, pid:1434, start at 2025-04-06 08:21:18.777520451 +0800 CST m=+2.491863020 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:23:18.786 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:21:18.812 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:11:18.812 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/availability/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/availability/tiflash/log/error.log arg matches is ArgMatches { args: {"config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/availability/tiflash-proxy.toml"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/availability/tiflash/db/proxy"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/availability/tiflash/log/proxy.log"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489988628702016704 TiDB Cluster ID. tikv_gc_leader_uuid 65821e494400007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-x-4tfw6, pid:1342, start at 2025-04-06 08:21:19.270696945 +0800 CST m=+2.762667901 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:23:19.282 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:21:19.248 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:11:19.248 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489988627476280682 TiDB Cluster ID. tikv_gc_leader_uuid 65821e4968c000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-x-4tfw6, pid:1415, start at 2025-04-06 08:21:19.413155663 +0800 CST m=+2.834034394 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:23:19.423 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:21:19.395 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:11:19.395 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored table partition_table.finish_mark exists check diff successfully pass test: get processor VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489988627685956005 TiDB Cluster ID. tikv_gc_leader_uuid 65821e489e8000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-j-x7t22, pid:1295, start at 2025-04-06 08:21:18.612538739 +0800 CST m=+2.124921223 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:23:18.660 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:21:18.653 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:11:18.653 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489988627685956005 TiDB Cluster ID. tikv_gc_leader_uuid 65821e489e8000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-j-x7t22, pid:1295, start at 2025-04-06 08:21:18.612538739 +0800 CST m=+2.124921223 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:23:18.660 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:21:18.653 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:11:18.653 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489988626394389026 TiDB Cluster ID. tikv_gc_leader_uuid 65821e48d08000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-j-x7t22, pid:1355, start at 2025-04-06 08:21:18.814615574 +0800 CST m=+2.246490410 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:23:18.853 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:21:18.853 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:11:18.853 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/clustered_index/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/clustered_index/tiflash/log/error.log arg matches is ArgMatches { args: {"addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/clustered_index/tiflash/log/proxy.log"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/clustered_index/tiflash-proxy.toml"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/clustered_index/tiflash/db/proxy"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/tiflash/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/tiflash/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/tiflash/tiflash-proxy.toml"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/tiflash/tiflash/db/proxy"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/tiflash/tiflash/log/proxy.log"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Changefeed remove successfully. ID: changefeed-error-2 CheckpointTs: 457152616123858945 SinkURI: mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1 PASS wait process cdc.test exit for 1-th time... Starting Upstream TiDB... coverage: 2.0% of statements in github.com/pingcap/tiflow/... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489988630156907620 TiDB Cluster ID. tikv_gc_leader_uuid 65821e495bc000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-x-hbn5c, pid:1282, start at 2025-04-06 08:21:19.358868372 +0800 CST m=+2.268645926 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:23:19.366 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:21:19.343 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:11:19.343 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489988630156907620 TiDB Cluster ID. tikv_gc_leader_uuid 65821e495bc000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-x-hbn5c, pid:1282, start at 2025-04-06 08:21:19.358868372 +0800 CST m=+2.268645926 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:23:19.366 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:21:19.343 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:11:19.343 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489988634502908368 TiDB Cluster ID. tikv_gc_leader_uuid 65821e4966c0004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-x-hbn5c, pid:1358, start at 2025-04-06 08:21:19.392697018 +0800 CST m=+2.222875461 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:23:19.400 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:21:19.387 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:11:19.387 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/new_ci_collation/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/new_ci_collation/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/new_ci_collation/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/new_ci_collation/tiflash/log/proxy.log"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/new_ci_collation/tiflash/db/proxy"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [Sun Apr 6 08:21:21 CST 2025] <<<<<< START cdc server in changefeed_finish case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_finish.26762678.out server --log-file /tmp/tidb_cdc_test/changefeed_finish/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_finish/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489988635534264163 TiDB Cluster ID. tikv_gc_leader_uuid 65821e4a0d40007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-c-g3td9, pid:1355, start at 2025-04-06 08:21:20.06364803 +0800 CST m=+2.502841136 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:23:20.072 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:21:20.053 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:11:20.053 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489988635534264163 TiDB Cluster ID. tikv_gc_leader_uuid 65821e4a0d40007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-c-g3td9, pid:1355, start at 2025-04-06 08:21:20.06364803 +0800 CST m=+2.502841136 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:23:20.072 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:21:20.053 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:11:20.053 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489988632646484198 TiDB Cluster ID. tikv_gc_leader_uuid 65821e4a3440006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-c-g3td9, pid:1416, start at 2025-04-06 08:21:20.237447178 +0800 CST m=+2.591876422 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:23:20.246 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:21:20.209 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:11:20.209 +0800 All versions after safe point can be accessed. (DO NOT EDIT) wait process cdc.test exit for 2-th time... Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/gc_safepoint/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/gc_safepoint/tiflash/log/error.log arg matches is ArgMatches { args: {"memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/gc_safepoint/tiflash/log/proxy.log"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/gc_safepoint/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/gc_safepoint/tiflash/db/proxy"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table ddl_manager.finish_mark not exists for 29-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489988633107885957 TiDB Cluster ID. tikv_gc_leader_uuid 65821e4a9800004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-x-pbmg2, pid:1351, start at 2025-04-06 08:21:20.63226093 +0800 CST m=+2.913404040 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:23:20.642 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:21:20.608 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:11:20.608 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489988633107885957 TiDB Cluster ID. tikv_gc_leader_uuid 65821e4a9800004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-x-pbmg2, pid:1351, start at 2025-04-06 08:21:20.63226093 +0800 CST m=+2.913404040 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:23:20.642 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:21:20.608 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:11:20.608 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489988637902554063 TiDB Cluster ID. tikv_gc_leader_uuid 65821e4ab140001 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-x-pbmg2, pid:1431, start at 2025-04-06 08:21:20.726072764 +0800 CST m=+2.947093506 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:23:20.736 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:21:20.709 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:11:20.709 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/bank/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/bank/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/bank/tiflash-proxy.toml"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/bank/tiflash/log/proxy.log"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/bank/tiflash/db/proxy"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } pass test: move table ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Sun Apr 6 08:21:22 CST 2025] <<<<<< run test case partition_table success! >>>>>> Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Sun Apr 6 08:21:22 CST 2025] <<<<<< START cdc server in owner_resign case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.owner_resign.26042606.out server --log-file /tmp/tidb_cdc_test/owner_resign/cdc1.log --log-level debug --data-dir /tmp/tidb_cdc_test/owner_resign/cdc_data1 --cluster-id default --addr 127.0.0.1:8301 --pd http://127.0.0.1:2379 + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.cli.2682.out cli tso query --pd=http://127.0.0.1:2379 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.simple.cli.2661.out cli tso query --pd=http://127.0.0.1:2379 + set +x [Sun Apr 6 08:21:22 CST 2025] <<<<<< START cdc server in clustered_index case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.clustered_index.26362638.out server --log-file /tmp/tidb_cdc_test/clustered_index/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/clustered_index/cdc_data --cluster-id default + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.tiflash.cli.2667.out cli tso query --pd=http://127.0.0.1:2379 wait process cdc.test exit for 1-th time... pass test: set log level wait process cdc.test exit for 2-th time... [Sun Apr 6 08:21:23 CST 2025] <<<<<< START cdc server in gc_safepoint case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS='github.com/pingcap/tiflow/pkg/txnutil/gc/InjectGcSafepointUpdateInterval=return(500)' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.gc_safepoint.26612663.out server --log-file /tmp/tidb_cdc_test/gc_safepoint/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/gc_safepoint/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table ddl_manager.finish_mark not exists for 30-th check, retry later [Sun Apr 6 08:21:23 CST 2025] <<<<<< START cdc server in new_ci_collation case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.new_ci_collation.27182720.out server --log-file /tmp/tidb_cdc_test/new_ci_collation/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/new_ci_collation/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + set +x + tso='457152628682129409 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 457152628682129409 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Sun Apr 6 08:21:24 CST 2025] <<<<<< START cdc server in simple case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.simple.27142716.out server --log-file /tmp/tidb_cdc_test/simple/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/simple/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 06 Apr 2025 00:21:24 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/74c207ed-a166-40e7-bb55-2d8f79ba1bc2 {"id":"74c207ed-a166-40e7-bb55-2d8f79ba1bc2","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898881} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319608790185d8 74c207ed-a166-40e7-bb55-2d8f79ba1bc2 /tidb/cdc/default/default/upstream/7489988621899277058 {"id":7489988621899277058,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/74c207ed-a166-40e7-bb55-2d8f79ba1bc2 {"id":"74c207ed-a166-40e7-bb55-2d8f79ba1bc2","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898881} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319608790185d8 74c207ed-a166-40e7-bb55-2d8f79ba1bc2 /tidb/cdc/default/default/upstream/7489988621899277058 {"id":7489988621899277058,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/74c207ed-a166-40e7-bb55-2d8f79ba1bc2 {"id":"74c207ed-a166-40e7-bb55-2d8f79ba1bc2","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898881} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319608790185d8 74c207ed-a166-40e7-bb55-2d8f79ba1bc2 /tidb/cdc/default/default/upstream/7489988621899277058 {"id":7489988621899277058,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Sun Apr 6 08:21:24 CST 2025] <<<<<< START cdc server in changefeed_error case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/redo/ChangefeedNewRedoManagerError=2*return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.63816383.out server --log-file /tmp/tidb_cdc_test/changefeed_error/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_error/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x + tso='457152628763131907 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 457152628763131907 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x run test case test_kill_owner [Sun Apr 6 08:21:24 CST 2025] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.27392741.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_kill_owner.server1.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_kill_owner.server1 --cluster-id default + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489988645087467134 TiDB Cluster ID. tikv_gc_leader_uuid 65821e4ca9c0035 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-j-vgmw9, pid:1404, start at 2025-04-06 08:21:22.743892511 +0800 CST m=+2.545952456 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:23:22.751 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:21:22.752 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:11:22.752 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489988645087467134 TiDB Cluster ID. tikv_gc_leader_uuid 65821e4ca9c0035 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-j-vgmw9, pid:1404, start at 2025-04-06 08:21:22.743892511 +0800 CST m=+2.545952456 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:23:22.751 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:21:22.752 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:11:22.752 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489988634872971079 TiDB Cluster ID. tikv_gc_leader_uuid 65821e4cd4c000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-j-vgmw9, pid:1471, start at 2025-04-06 08:21:22.943674203 +0800 CST m=+2.686267021 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:23:22.952 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:21:22.949 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:11:22.949 +0800 All versions after safe point can be accessed. (DO NOT EDIT) [Sun Apr 6 08:21:24 CST 2025] <<<<<< START cdc server in bank case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.bank.26972699.out server --log-file /tmp/tidb_cdc_test/bank/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/bank/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [Pipeline] stage + set +x + tso='457152628863270913 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 457152628863270913 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Sun Apr 6 08:21:24 CST 2025] <<<<<< START cdc server in tiflash case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.tiflash.27022704.out server --log-file /tmp/tidb_cdc_test/tiflash/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/tiflash/cdc_data --cluster-id default + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [Pipeline] { (Test) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/resolve_lock/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/resolve_lock/tiflash/log/error.log arg matches is ArgMatches { args: {"memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/resolve_lock/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/resolve_lock/tiflash/log/proxy.log"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/resolve_lock/tiflash/db/proxy"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489988638189900455 TiDB Cluster ID. tikv_gc_leader_uuid 65821e4c7a0000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-4-tjmd8, pid:1368, start at 2025-04-06 08:21:22.582436744 +0800 CST m=+2.200382562 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:23:22.591 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:21:22.586 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:11:22.586 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489988638189900455 TiDB Cluster ID. tikv_gc_leader_uuid 65821e4c7a0000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-4-tjmd8, pid:1368, start at 2025-04-06 08:21:22.582436744 +0800 CST m=+2.200382562 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:23:22.591 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:21:22.586 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:11:22.586 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489988646917486626 TiDB Cluster ID. tikv_gc_leader_uuid 65821e4c938000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-4-tjmd8, pid:1444, start at 2025-04-06 08:21:22.679663151 +0800 CST m=+2.229961916 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:23:22.687 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:21:22.688 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:11:22.688 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/many_pk_or_uk/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/many_pk_or_uk/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/many_pk_or_uk/tiflash/db/proxy"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/many_pk_or_uk/tiflash-proxy.toml"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/many_pk_or_uk/tiflash/log/proxy.log"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } pass test: remove changefeed check diff failed 1-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 06 Apr 2025 00:21:25 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ddf0664d-5b64-470b-ad98-6496a2b72978 {"id":"ddf0664d-5b64-470b-ad98-6496a2b72978","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898882} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319608790502d6 ddf0664d-5b64-470b-ad98-6496a2b72978 /tidb/cdc/default/default/upstream/7489988629164751674 {"id":7489988629164751674,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ddf0664d-5b64-470b-ad98-6496a2b72978 {"id":"ddf0664d-5b64-470b-ad98-6496a2b72978","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898882} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319608790502d6 ddf0664d-5b64-470b-ad98-6496a2b72978 /tidb/cdc/default/default/upstream/7489988629164751674 {"id":7489988629164751674,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ddf0664d-5b64-470b-ad98-6496a2b72978 {"id":"ddf0664d-5b64-470b-ad98-6496a2b72978","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898882} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319608790502d6 ddf0664d-5b64-470b-ad98-6496a2b72978 /tidb/cdc/default/default/upstream/7489988629164751674 {"id":7489988629164751674,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x cdc cli capture list --server http://127.0.0.1:8301 |jq '.|length'|grep -E '^1$' 1 run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.owner_resign.cli.2697.out cli changefeed create --sink-uri=mysql://normal:123456@127.0.0.1:3306/ --server=127.0.0.1:8301 table ddl_manager.finish_mark not exists for 31-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489988650781543774 TiDB Cluster ID. tikv_gc_leader_uuid 65821e4e94c0009 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-z-4sl2f, pid:1345, start at 2025-04-06 08:21:24.707717294 +0800 CST m=+3.159976841 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:23:24.736 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:21:24.741 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:11:24.741 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489988650781543774 TiDB Cluster ID. tikv_gc_leader_uuid 65821e4e94c0009 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-z-4sl2f, pid:1345, start at 2025-04-06 08:21:24.707717294 +0800 CST m=+3.159976841 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:23:24.736 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:21:24.741 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:11:24.741 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489988650175228904 TiDB Cluster ID. tikv_gc_leader_uuid 65821e4eb980007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-z-4sl2f, pid:1413, start at 2025-04-06 08:21:24.848593168 +0800 CST m=+3.190536149 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:23:24.857 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:21:24.838 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:11:24.838 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/tidb_mysql_test/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/tidb_mysql_test/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/tidb_mysql_test/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/tidb_mysql_test/tiflash/db/proxy"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/tidb_mysql_test/tiflash/log/proxy.log"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 06 Apr 2025 00:21:26 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5cb4f656-2325-4bd8-b4ee-11051bdce719 {"id":"5cb4f656-2325-4bd8-b4ee-11051bdce719","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898883} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319608790655d6 5cb4f656-2325-4bd8-b4ee-11051bdce719 /tidb/cdc/default/default/upstream/7489988627685956005 {"id":7489988627685956005,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5cb4f656-2325-4bd8-b4ee-11051bdce719 {"id":"5cb4f656-2325-4bd8-b4ee-11051bdce719","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898883} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319608790655d6 5cb4f656-2325-4bd8-b4ee-11051bdce719 /tidb/cdc/default/default/upstream/7489988627685956005 {"id":7489988627685956005,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5cb4f656-2325-4bd8-b4ee-11051bdce719 {"id":"5cb4f656-2325-4bd8-b4ee-11051bdce719","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898883} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319608790655d6 5cb4f656-2325-4bd8-b4ee-11051bdce719 /tidb/cdc/default/default/upstream/7489988627685956005 {"id":7489988627685956005,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: 0032d73c-e25b-4527-aac4-a794c62b81c4 Info: {"upstream_id":7489988627685956005,"namespace":"default","id":"0032d73c-e25b-4527-aac4-a794c62b81c4","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-06T08:21:26.157159105+08:00","start_ts":457152628768374785,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457152628768374785,"checkpoint_ts":457152628768374785,"checkpoint_time":"2025-04-06 08:21:22.936"} VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489988644601760856 TiDB Cluster ID. tikv_gc_leader_uuid 65821e4e8c80012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-d-7dnbd, pid:9900, start at 2025-04-06 08:21:24.699387002 +0800 CST m=+2.645911641 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:23:24.707 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:21:24.708 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:11:24.708 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489988644601760856 TiDB Cluster ID. tikv_gc_leader_uuid 65821e4e8c80012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-d-7dnbd, pid:9900, start at 2025-04-06 08:21:24.699387002 +0800 CST m=+2.645911641 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:23:24.707 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:21:24.708 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:11:24.708 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489988650882337163 TiDB Cluster ID. tikv_gc_leader_uuid 65821e4ea540009 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-d-7dnbd, pid:9975, start at 2025-04-06 08:21:24.774287509 +0800 CST m=+2.663313833 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:23:24.783 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:21:24.757 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:11:24.757 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... Create changefeed successfully! ID: 9059244d-df63-4544-be0d-9cc978a38a33 Info: {"upstream_id":7489988629164751674,"namespace":"default","id":"9059244d-df63-4544-be0d-9cc978a38a33","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-06T08:21:26.135526297+08:00","start_ts":457152629583904770,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457152629583904770,"checkpoint_ts":457152629583904770,"checkpoint_time":"2025-04-06 08:21:26.047"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... pass test: resign owner + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/synced_status_with_redo/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/synced_status_with_redo/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/synced_status_with_redo/tiflash/log/proxy.log"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/synced_status_with_redo/tiflash-proxy.toml"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/synced_status_with_redo/tiflash/db/proxy"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.resolve_lock.cli.2716.out cli tso query --pd=http://127.0.0.1:2379 < HTTP/1.1 200 OK < Date: Sun, 06 Apr 2025 00:21:26 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7d429b92-7697-4321-aeb8-d9df3460b8f4 {"id":"7d429b92-7697-4321-aeb8-d9df3460b8f4","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898883} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087904c9d4 7d429b92-7697-4321-aeb8-d9df3460b8f4 /tidb/cdc/default/default/upstream/7489988630156907620 {"id":7489988630156907620,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7d429b92-7697-4321-aeb8-d9df3460b8f4 {"id":"7d429b92-7697-4321-aeb8-d9df3460b8f4","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898883} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087904c9d4 7d429b92-7697-4321-aeb8-d9df3460b8f4 /tidb/cdc/default/default/upstream/7489988630156907620 {"id":7489988630156907620,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7d429b92-7697-4321-aeb8-d9df3460b8f4 {"id":"7d429b92-7697-4321-aeb8-d9df3460b8f4","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898883} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087904c9d4 7d429b92-7697-4321-aeb8-d9df3460b8f4 /tidb/cdc/default/default/upstream/7489988630156907620 {"id":7489988630156907620,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: 7e6f77d5-2313-4415-9f4a-e0e59c3a5085 Info: {"upstream_id":7489988630156907620,"namespace":"default","id":"7e6f77d5-2313-4415-9f4a-e0e59c3a5085","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/?safe-mode=true","create_time":"2025-04-06T08:21:26.782891404+08:00","start_ts":457152628927758337,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457152628927758337,"checkpoint_ts":457152628927758337,"checkpoint_time":"2025-04-06 08:21:23.544"} + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.many_pk_or_uk.cli.2756.out cli tso query --pd=http://127.0.0.1:2379 check diff failed 2-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 06 Apr 2025 00:21:27 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d7e8f845-f9f2-4c1d-9685-25bcf6acd8ad {"id":"d7e8f845-f9f2-4c1d-9685-25bcf6acd8ad","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898884} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319608790697d3 d7e8f845-f9f2-4c1d-9685-25bcf6acd8ad /tidb/cdc/default/default/upstream/7489988635534264163 {"id":7489988635534264163,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d7e8f845-f9f2-4c1d-9685-25bcf6acd8ad {"id":"d7e8f845-f9f2-4c1d-9685-25bcf6acd8ad","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898884} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319608790697d3 d7e8f845-f9f2-4c1d-9685-25bcf6acd8ad /tidb/cdc/default/default/upstream/7489988635534264163 {"id":7489988635534264163,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d7e8f845-f9f2-4c1d-9685-25bcf6acd8ad {"id":"d7e8f845-f9f2-4c1d-9685-25bcf6acd8ad","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898884} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319608790697d3 d7e8f845-f9f2-4c1d-9685-25bcf6acd8ad /tidb/cdc/default/default/upstream/7489988635534264163 {"id":7489988635534264163,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x pass test: get tso table new_ci_collation_test.t1 not exists for 1-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 06 Apr 2025 00:21:27 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/face3b44-90af-44f1-9a40-f36299946baa {"id":"face3b44-90af-44f1-9a40-f36299946baa","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898884} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319608790378fb face3b44-90af-44f1-9a40-f36299946baa /tidb/cdc/default/default/upstream/7489988621880289090 {"id":7489988621880289090,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/face3b44-90af-44f1-9a40-f36299946baa {"id":"face3b44-90af-44f1-9a40-f36299946baa","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898884} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319608790378fb face3b44-90af-44f1-9a40-f36299946baa /tidb/cdc/default/default/upstream/7489988621880289090 {"id":7489988621880289090,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/face3b44-90af-44f1-9a40-f36299946baa {"id":"face3b44-90af-44f1-9a40-f36299946baa","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898884} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319608790378fb face3b44-90af-44f1-9a40-f36299946baa /tidb/cdc/default/default/upstream/7489988621880289090 {"id":7489988621880289090,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.simple.cli.2773.out cli changefeed create --start-ts=457152628682129409 --sink-uri=mysql+ssl://normal:123456@127.0.0.1:3306/ 0 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 06 Apr 2025 00:21:27 GMT < Content-Length: 845 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/22ea5c43-0c61-4454-a125-5e15e38912dd {"id":"22ea5c43-0c61-4454-a125-5e15e38912dd","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898884} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 7 /tidb/cdc/default/__cdc_meta__/owner/223196087848f21e 22ea5c43-0c61-4454-a125-5e15e38912dd /tidb/cdc/default/default/upstream/7489988418452591948 {"id":7489988418452591948,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/22ea5c43-0c61-4454-a125-5e15e38912dd {"id":"22ea5c43-0c61-4454-a125-5e15e38912dd","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898884} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 7 /tidb/cdc/default/__cdc_meta__/owner/223196087848f21e 22ea5c43-0c61-4454-a125-5e15e38912dd /tidb/cdc/default/default/upstream/7489988418452591948 {"id":7489988418452591948,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/22ea5c43-0c61-4454-a125-5e15e38912dd {"id":"22ea5c43-0c61-4454-a125-5e15e38912dd","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898884} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 7 /tidb/cdc/default/__cdc_meta__/owner/223196087848f21e 22ea5c43-0c61-4454-a125-5e15e38912dd /tidb/cdc/default/default/upstream/7489988418452591948 {"id":7489988418452591948,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.6436.out cli changefeed create --start-ts=0 '--sink-uri=mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1' -c changefeed-initialize-error Create changefeed successfully! ID: 82c4ffc2-4ae1-4e88-b5c4-3519d05eed47 Info: {"upstream_id":7489988621880289090,"namespace":"default","id":"82c4ffc2-4ae1-4e88-b5c4-3519d05eed47","sink_uri":"mysql+ssl://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-06T08:21:27.691446228+08:00","start_ts":457152628682129409,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457152628682129409,"checkpoint_ts":457152628682129409,"checkpoint_time":"2025-04-06 08:21:22.607"} PASS + set +x cdc cli processor list --server http://127.0.0.1:8301 |jq '.|length'|grep -E '^1$' 1 run task successfully Create changefeed successfully! ID: changefeed-initialize-error Info: {"upstream_id":7489988418452591948,"namespace":"default","id":"changefeed-initialize-error","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create_time":"2025-04-06T08:21:27.769459237+08:00","start_ts":457152630004645893,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457152630004645893,"checkpoint_ts":457152630004645893,"checkpoint_time":"2025-04-06 08:21:27.652"} PASS + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 06 Apr 2025 00:21:27 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/afe53a47-596b-4147-92c4-b144550f6349 {"id":"afe53a47-596b-4147-92c4-b144550f6349","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898884} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319608790544fd afe53a47-596b-4147-92c4-b144550f6349 /tidb/cdc/default/default/upstream/7489988628380945915 {"id":7489988628380945915,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/afe53a47-596b-4147-92c4-b144550f6349 {"id":"afe53a47-596b-4147-92c4-b144550f6349","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898884} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319608790544fd afe53a47-596b-4147-92c4-b144550f6349 /tidb/cdc/default/default/upstream/7489988628380945915 {"id":7489988628380945915,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/afe53a47-596b-4147-92c4-b144550f6349 {"id":"afe53a47-596b-4147-92c4-b144550f6349","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898884} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319608790544fd afe53a47-596b-4147-92c4-b144550f6349 /tidb/cdc/default/default/upstream/7489988628380945915 {"id":7489988628380945915,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.cli.2794.out cli changefeed create --start-ts=457152628763131907 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ coverage: 2.4% of statements in github.com/pingcap/tiflow/... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 06 Apr 2025 00:21:27 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1d38cc91-1521-420f-9ccf-dba46fb23c09 {"id":"1d38cc91-1521-420f-9ccf-dba46fb23c09","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898885} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319608790b1ae2 1d38cc91-1521-420f-9ccf-dba46fb23c09 /tidb/cdc/default/default/upstream/7489988633107885957 {"id":7489988633107885957,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1d38cc91-1521-420f-9ccf-dba46fb23c09 {"id":"1d38cc91-1521-420f-9ccf-dba46fb23c09","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898885} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319608790b1ae2 1d38cc91-1521-420f-9ccf-dba46fb23c09 /tidb/cdc/default/default/upstream/7489988633107885957 {"id":7489988633107885957,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1d38cc91-1521-420f-9ccf-dba46fb23c09 {"id":"1d38cc91-1521-420f-9ccf-dba46fb23c09","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898885} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319608790b1ae2 1d38cc91-1521-420f-9ccf-dba46fb23c09 /tidb/cdc/default/default/upstream/7489988633107885957 {"id":7489988633107885957,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.bank.cli.2761.out cli changefeed create --sink-uri=mysql://root@127.0.0.1:3306/ table owner_resign.t1 not exists for 1-th check, retry later + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.tidb_mysql_test.cli.2742.out cli tso query --pd=http://127.0.0.1:2379 coverage: 2.4% of statements in github.com/pingcap/tiflow/... Create changefeed successfully! ID: e684662d-022f-4a79-9ef3-0a59dba96193 Info: {"upstream_id":7489988628380945915,"namespace":"default","id":"e684662d-022f-4a79-9ef3-0a59dba96193","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-06T08:21:28.032207553+08:00","start_ts":457152628763131907,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457152628763131907,"checkpoint_ts":457152628763131907,"checkpoint_time":"2025-04-06 08:21:22.916"} PASS + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 06 Apr 2025 00:21:27 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c1d19483-f551-4785-88c5-e5db23952559 {"id":"c1d19483-f551-4785-88c5-e5db23952559","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898885} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319608790a46e3 c1d19483-f551-4785-88c5-e5db23952559 /tidb/cdc/default/default/upstream/7489988628702016704 {"id":7489988628702016704,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c1d19483-f551-4785-88c5-e5db23952559 {"id":"c1d19483-f551-4785-88c5-e5db23952559","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898885} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319608790a46e3 c1d19483-f551-4785-88c5-e5db23952559 /tidb/cdc/default/default/upstream/7489988628702016704 {"id":7489988628702016704,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c1d19483-f551-4785-88c5-e5db23952559 {"id":"c1d19483-f551-4785-88c5-e5db23952559","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898885} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319608790a46e3 c1d19483-f551-4785-88c5-e5db23952559 /tidb/cdc/default/default/upstream/7489988628702016704 {"id":7489988628702016704,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: 3adf97ae-e937-4f06-894b-c9870d4ac5ee Info: {"upstream_id":7489988628702016704,"namespace":"default","id":"3adf97ae-e937-4f06-894b-c9870d4ac5ee","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-06T08:21:27.980160691+08:00","start_ts":457152628863270913,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457152628863270913,"checkpoint_ts":457152628863270913,"checkpoint_time":"2025-04-06 08:21:23.298"} coverage: 2.4% of statements in github.com/pingcap/tiflow/... table ddl_manager.finish_mark not exists for 32-th check, retry later + cd /tmp/tidb_cdc_test/synced_status_with_redo ++ run_cdc_cli_tso_query 127.0.0.1 2379 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status_with_redo.cli.11353.out cli tso query --pd=http://127.0.0.1:2379 Create changefeed successfully! ID: ec9eab50-1ea6-4972-8487-0f49c43187c4 Info: {"upstream_id":7489988633107885957,"namespace":"default","id":"ec9eab50-1ea6-4972-8487-0f49c43187c4","sink_uri":"mysql://root@127.0.0.1:3306/","create_time":"2025-04-06T08:21:28.24595829+08:00","start_ts":457152630137290754,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457152630137290754,"checkpoint_ts":457152630137290754,"checkpoint_time":"2025-04-06 08:21:28.158"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... pass test: verify table + set +x + tso='457152629886943233 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 457152629886943233 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Sun Apr 6 08:21:28 CST 2025] <<<<<< START cdc server in resolve_lock case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.resolve_lock.27522754.out server --log-file /tmp/tidb_cdc_test/resolve_lock/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/resolve_lock/cdc_data --cluster-id default + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + set +x + tso='457152629856272385 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 457152629856272385 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Sun Apr 6 08:21:28 CST 2025] <<<<<< START cdc server in many_pk_or_uk case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.many_pk_or_uk.27932795.out server --log-file /tmp/tidb_cdc_test/many_pk_or_uk/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/many_pk_or_uk/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff failed 1-th time, retry later table new_ci_collation_test.t1 exists table new_ci_collation_test.t2 not exists for 1-th check, retry later + set +x succeed to verify meta placement rules + set +x check_changefeed_state http://127.0.0.1:2379 changefeed-initialize-error normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-initialize-error + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-initialize-error -s check diff successfully TEST FAILED: OUTPUT DOES NOT CONTAIN 'id: 1' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ check data failed 1-th time, retry later + info='{ "upstream_id": 7489988418452591948, "namespace": "default", "id": "changefeed-initialize-error", "state": "warning", "checkpoint_tso": 457152630004645893, "checkpoint_time": "2025-04-06 08:21:27.652", "error": { "time": "2025-04-06T08:21:27.954509841+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } }' + echo '{ "upstream_id": 7489988418452591948, "namespace": "default", "id": "changefeed-initialize-error", "state": "warning", "checkpoint_tso": 457152630004645893, "checkpoint_time": "2025-04-06 08:21:27.652", "error": { "time": "2025-04-06T08:21:27.954509841+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } }' { "upstream_id": 7489988418452591948, "namespace": "default", "id": "changefeed-initialize-error", "state": "warning", "checkpoint_tso": 457152630004645893, "checkpoint_time": "2025-04-06 08:21:27.652", "error": { "time": "2025-04-06T08:21:27.954509841+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } } ++ echo '{' '"upstream_id":' 7489988418452591948, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"warning",' '"checkpoint_tso":' 457152630004645893, '"checkpoint_time":' '"2025-04-06' '08:21:27.652",' '"error":' '{' '"time":' '"2025-04-06T08:21:27.954509841+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrProcessorUnknown",' '"message":' '"changefeed' new redo manager injected 'error"' '}' '}' ++ jq -r .state + state=warning + [[ ! warning == \n\o\r\m\a\l ]] + echo 'changefeed state warning does not equal to normal' changefeed state warning does not equal to normal + exit 1 run task failed 1-th time, retry later + set +x cdc.test cli capture list 2>&1 | grep '"is-owner": true' table owner_resign.t1 exists + set +x + tso='457152630185263106 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 457152630185263106 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Sun Apr 6 08:21:29 CST 2025] <<<<<< START cdc server in tidb_mysql_test case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.tidb_mysql_test.27772779.out server --log-file /tmp/tidb_cdc_test/tidb_mysql_test/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/tidb_mysql_test/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + set +x [Sun Apr 6 08:21:29 CST 2025] <<<<<< START cdc server in bank case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8400/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.bank.27862788.out server --log-file /tmp/tidb_cdc_test/bank/cdcdown.log --log-level debug --data-dir /tmp/tidb_cdc_test/bank/cdc_datadown --cluster-id default --addr 127.0.0.1:8400 --pd http://127.0.0.1:2479 ++ curl -vsL --max-time 20 http://127.0.0.1:8400/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8400 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8400; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table cdc_tiflash_test.multi_data_type not exists for 1-th check, retry later pass test: create changefeed v2 check diff failed 1-th time, retry later [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow [Pipeline] { [Pipeline] cache table ddl_manager.finish_mark not exists for 33-th check, retry later + set +x + tso='457152630307684353 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 457152630307684353 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + start_ts=457152630307684353 + run_cdc_server --workdir /tmp/tidb_cdc_test/synced_status_with_redo --binary cdc.test [Sun Apr 6 08:21:30 CST 2025] <<<<<< START cdc server in synced_status_with_redo case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status_with_redo.1138511387.out server --log-file /tmp/tidb_cdc_test/synced_status_with_redo/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/synced_status_with_redo/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff failed 2-th time, retry later "is-owner": true, run task successfully check_changefeed_state http://127.0.0.1:2379 changefeed-initialize-error normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-initialize-error + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-initialize-error -s + info='{ "upstream_id": 7489988418452591948, "namespace": "default", "id": "changefeed-initialize-error", "state": "warning", "checkpoint_tso": 457152630004645893, "checkpoint_time": "2025-04-06 08:21:27.652", "error": { "time": "2025-04-06T08:21:27.954509841+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } }' + echo '{ "upstream_id": 7489988418452591948, "namespace": "default", "id": "changefeed-initialize-error", "state": "warning", "checkpoint_tso": 457152630004645893, "checkpoint_time": "2025-04-06 08:21:27.652", "error": { "time": "2025-04-06T08:21:27.954509841+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } }' { "upstream_id": 7489988418452591948, "namespace": "default", "id": "changefeed-initialize-error", "state": "warning", "checkpoint_tso": 457152630004645893, "checkpoint_time": "2025-04-06 08:21:27.652", "error": { "time": "2025-04-06T08:21:27.954509841+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } } ++ echo '{' '"upstream_id":' 7489988418452591948, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"warning",' '"checkpoint_tso":' 457152630004645893, '"checkpoint_time":' '"2025-04-06' '08:21:27.652",' '"error":' '{' '"time":' '"2025-04-06T08:21:27.954509841+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrProcessorUnknown",' '"message":' '"changefeed' new redo manager injected 'error"' '}' '}' ++ jq -r .state + state=warning + [[ ! warning == \n\o\r\m\a\l ]] + echo 'changefeed state warning does not equal to normal' changefeed state warning does not equal to normal + exit 1 run task failed 2-th time, retry later table new_ci_collation_test.t2 exists table new_ci_collation_test.t3 not exists for 1-th check, retry later pass test: remove changefeed v2 table cdc_tiflash_test.multi_data_type exists check diff failed 1-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 06 Apr 2025 00:21:31 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0c1b768d-d1e7-4eea-bc92-6bab213f2ccf {"id":"0c1b768d-d1e7-4eea-bc92-6bab213f2ccf","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898889} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087910f0e8 0c1b768d-d1e7-4eea-bc92-6bab213f2ccf /tidb/cdc/default/default/upstream/7489988645087467134 {"id":7489988645087467134,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0c1b768d-d1e7-4eea-bc92-6bab213f2ccf {"id":"0c1b768d-d1e7-4eea-bc92-6bab213f2ccf","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898889} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087910f0e8 0c1b768d-d1e7-4eea-bc92-6bab213f2ccf /tidb/cdc/default/default/upstream/7489988645087467134 {"id":7489988645087467134,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0c1b768d-d1e7-4eea-bc92-6bab213f2ccf {"id":"0c1b768d-d1e7-4eea-bc92-6bab213f2ccf","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898889} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087910f0e8 0c1b768d-d1e7-4eea-bc92-6bab213f2ccf /tidb/cdc/default/default/upstream/7489988645087467134 {"id":7489988645087467134,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.resolve_lock.cli.2803.out cli changefeed create --start-ts=457152629886943233 --sink-uri=mysql://normal:123456@127.0.0.1:3306/tidb-txn-mode=pessimistic + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 06 Apr 2025 00:21:31 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ecb5c13b-9d4a-4401-a217-67228e5e6df7 {"id":"ecb5c13b-9d4a-4401-a217-67228e5e6df7","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898888} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/2231960879155ee9 ecb5c13b-9d4a-4401-a217-67228e5e6df7 /tidb/cdc/default/default/upstream/7489988638189900455 {"id":7489988638189900455,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ecb5c13b-9d4a-4401-a217-67228e5e6df7 {"id":"ecb5c13b-9d4a-4401-a217-67228e5e6df7","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898888} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/2231960879155ee9 ecb5c13b-9d4a-4401-a217-67228e5e6df7 /tidb/cdc/default/default/upstream/7489988638189900455 {"id":7489988638189900455,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ecb5c13b-9d4a-4401-a217-67228e5e6df7 {"id":"ecb5c13b-9d4a-4401-a217-67228e5e6df7","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898888} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/2231960879155ee9 ecb5c13b-9d4a-4401-a217-67228e5e6df7 /tidb/cdc/default/default/upstream/7489988638189900455 {"id":7489988638189900455,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.many_pk_or_uk.cli.2847.out cli changefeed create --start-ts=457152629856272385 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ check diff successfully TEST FAILED: OUTPUT DOES NOT CONTAIN 'id: 1' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ check data failed 2-th time, retry later Create changefeed successfully! ID: 127674ed-885c-4d31-be17-c61541a27f4d table ddl_manager.finish_mark not exists for 34-th check, retry later Info: {"upstream_id":7489988645087467134,"namespace":"default","id":"127674ed-885c-4d31-be17-c61541a27f4d","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/tidb-txn-mode=pessimistic","create_time":"2025-04-06T08:21:32.124430778+08:00","start_ts":457152629886943233,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457152629886943233,"checkpoint_ts":457152629886943233,"checkpoint_time":"2025-04-06 08:21:27.203"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... Create changefeed successfully! ID: 975bfa50-dc1c-4e14-81ea-35fda75a01ef Info: {"upstream_id":7489988638189900455,"namespace":"default","id":"975bfa50-dc1c-4e14-81ea-35fda75a01ef","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-06T08:21:32.056728281+08:00","start_ts":457152629856272385,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457152629856272385,"checkpoint_ts":457152629856272385,"checkpoint_time":"2025-04-06 08:21:27.086"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... [Sun Apr 6 08:21:32 CST 2025] <<<<<< START cdc server in owner_resign case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ChangefeedOwnerDontUpdateCheckpoint=return(true)' + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.owner_resign.28492851.out server --log-file /tmp/tidb_cdc_test/owner_resign/cdc2.log --log-level debug --data-dir /tmp/tidb_cdc_test/owner_resign/cdc_data2 --cluster-id default --addr 127.0.0.1:8302 --pd http://127.0.0.1:2379 + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8302; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 owner pid: 2744 owner id afe53a47-596b-4147-92c4-b144550f6349 [Sun Apr 6 08:21:32 CST 2025] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.29022904.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_kill_owner.server2.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_kill_owner.server2 --cluster-id default --addr 127.0.0.1:8301 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/ddl_with_exists/run.sh using Sink-Type: mysql... <<================= [Sun Apr 6 08:21:32 CST 2025] <<<<<< run test case ddl_with_exists success! >>>>>> status code 200 pass test: show metadata status code 200 status code 200 pass test: delete service_gc_safepoint status code 500 pass test: resolve lock + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8400/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8400 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8400 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8400 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 06 Apr 2025 00:21:32 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/221de1e5-0971-48f7-ab86-bd72698639a6 {"id":"221de1e5-0971-48f7-ab86-bd72698639a6","address":"127.0.0.1:8400","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898890} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/35119608790b4d0e 221de1e5-0971-48f7-ab86-bd72698639a6 /tidb/cdc/default/default/upstream/7489988637902554063 {"id":7489988637902554063,"pd-endpoints":"http://127.0.0.1:2479,http://127.0.0.1:2479","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/221de1e5-0971-48f7-ab86-bd72698639a6 {"id":"221de1e5-0971-48f7-ab86-bd72698639a6","address":"127.0.0.1:8400","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898890} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/35119608790b4d0e 221de1e5-0971-48f7-ab86-bd72698639a6 /tidb/cdc/default/default/upstream/7489988637902554063 {"id":7489988637902554063,"pd-endpoints":"http://127.0.0.1:2479,http://127.0.0.1:2479","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/221de1e5-0971-48f7-ab86-bd72698639a6 {"id":"221de1e5-0971-48f7-ab86-bd72698639a6","address":"127.0.0.1:8400","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898890} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/35119608790b4d0e 221de1e5-0971-48f7-ab86-bd72698639a6 /tidb/cdc/default/default/upstream/7489988637902554063 {"id":7489988637902554063,"pd-endpoints":"http://127.0.0.1:2479,http://127.0.0.1:2479","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.bank.cli.2859.out cli changefeed create --sink-uri=blackhole:// -c changefeed-for-find-finished-ts --server http://127.0.0.1:8400 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 06 Apr 2025 00:21:32 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/eaeb5e25-6408-4533-a901-f89aec9c0a59 {"id":"eaeb5e25-6408-4533-a901-f89aec9c0a59","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898890} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/2231960879162ae4 eaeb5e25-6408-4533-a901-f89aec9c0a59 /tidb/cdc/default/default/upstream/7489988650781543774 {"id":7489988650781543774,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/eaeb5e25-6408-4533-a901-f89aec9c0a59 {"id":"eaeb5e25-6408-4533-a901-f89aec9c0a59","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898890} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/2231960879162ae4 eaeb5e25-6408-4533-a901-f89aec9c0a59 /tidb/cdc/default/default/upstream/7489988650781543774 {"id":7489988650781543774,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/eaeb5e25-6408-4533-a901-f89aec9c0a59 {"id":"eaeb5e25-6408-4533-a901-f89aec9c0a59","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898890} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/2231960879162ae4 eaeb5e25-6408-4533-a901-f89aec9c0a59 /tidb/cdc/default/default/upstream/7489988650781543774 {"id":7489988650781543774,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 142 0 0 100 142 0 1360 --:--:-- --:--:-- --:--:-- 1365 mysql_test start go: downloading github.com/maxshuang/mysql-tester v0.0.2 check diff successfully check_safepoint_forward http://127.0.0.1:2379 7489988635534264163 457152631053484032 457152629874098178 Create changefeed successfully! ID: changefeed-for-find-finished-ts Info: {"upstream_id":7489988637902554063,"namespace":"default","id":"changefeed-for-find-finished-ts","sink_uri":"blackhole:","create_time":"2025-04-06T08:21:33.172893705+08:00","start_ts":457152631435165697,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457152631435165697,"checkpoint_ts":457152631435165697,"checkpoint_time":"2025-04-06 08:21:33.109"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x go: downloading go1.23.4 (linux/amd64) + set +x go: downloading go1.23.4 (linux/amd64) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 06 Apr 2025 00:21:33 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/219197fc-f993-4765-8260-1340c12a601a {"id":"219197fc-f993-4765-8260-1340c12a601a","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898890} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319608791be5e6 219197fc-f993-4765-8260-1340c12a601a /tidb/cdc/default/default/upstream/7489988644601760856 {"id":7489988644601760856,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/219197fc-f993-4765-8260-1340c12a601a {"id":"219197fc-f993-4765-8260-1340c12a601a","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898890} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319608791be5e6 219197fc-f993-4765-8260-1340c12a601a /tidb/cdc/default/default/upstream/7489988644601760856 {"id":7489988644601760856,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/219197fc-f993-4765-8260-1340c12a601a {"id":"219197fc-f993-4765-8260-1340c12a601a","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898890} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319608791be5e6 219197fc-f993-4765-8260-1340c12a601a /tidb/cdc/default/default/upstream/7489988644601760856 {"id":7489988644601760856,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + config_path=conf/changefeed-redo.toml + SINK_URI='mysql://root@127.0.0.1:3306/?max-txn-row=1' + run_cdc_cli changefeed create --start-ts=457152630307684353 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/synced_status_with_redo/conf/changefeed-redo.toml + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status_with_redo.cli.11441.out cli changefeed create --start-ts=457152630307684353 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/synced_status_with_redo/conf/changefeed-redo.toml table new_ci_collation_test.t3 exists table new_ci_collation_test.t4 not exists for 1-th check, retry later run task successfully Create changefeed successfully! ID: test-1 Info: {"upstream_id":7489988644601760856,"namespace":"default","id":"test-1","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2025-04-06T08:21:33.759957361+08:00","start_ts":457152630307684353,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"eventual","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"storage":"file:///tmp/tidb_cdc_test/synced_status/redo","use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":120,"checkpoint_interval":20}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457152630307684353,"checkpoint_ts":457152630307684353,"checkpoint_time":"2025-04-06 08:21:28.808"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... check data successfully check_changefeed_state http://127.0.0.1:2379 1ca16915-453d-4d6c-ae3b-5cef0432008b stopped null + endpoints=http://127.0.0.1:2379 + changefeed_id=1ca16915-453d-4d6c-ae3b-5cef0432008b + expected_state=stopped + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c 1ca16915-453d-4d6c-ae3b-5cef0432008b -s table ddl_manager.finish_mark not exists for 35-th check, retry later wait process cdc.test exit for 1-th time... + info='{ "upstream_id": 7489988635534264163, "namespace": "default", "id": "1ca16915-453d-4d6c-ae3b-5cef0432008b", "state": "stopped", "checkpoint_tso": 457152631578034177, "checkpoint_time": "2025-04-06 08:21:33.654", "error": null }' + echo '{ "upstream_id": 7489988635534264163, "namespace": "default", "id": "1ca16915-453d-4d6c-ae3b-5cef0432008b", "state": "stopped", "checkpoint_tso": 457152631578034177, "checkpoint_time": "2025-04-06 08:21:33.654", "error": null }' { "upstream_id": 7489988635534264163, "namespace": "default", "id": "1ca16915-453d-4d6c-ae3b-5cef0432008b", "state": "stopped", "checkpoint_tso": 457152631578034177, "checkpoint_time": "2025-04-06 08:21:33.654", "error": null } ++ echo '{' '"upstream_id":' 7489988635534264163, '"namespace":' '"default",' '"id":' '"1ca16915-453d-4d6c-ae3b-5cef0432008b",' '"state":' '"stopped",' '"checkpoint_tso":' 457152631578034177, '"checkpoint_time":' '"2025-04-06' '08:21:33.654",' '"error":' null '}' ++ jq -r .state + state=stopped + [[ ! stopped == \s\t\o\p\p\e\d ]] ++ echo '{' '"upstream_id":' 7489988635534264163, '"namespace":' '"default",' '"id":' '"1ca16915-453d-4d6c-ae3b-5cef0432008b",' '"state":' '"stopped",' '"checkpoint_tso":' 457152631578034177, '"checkpoint_time":' '"2025-04-06' '08:21:33.654",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully check_safepoint_equal http://127.0.0.1:2379 7489988635534264163 check diff successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 1-th time... + set +x + GO111MODULE=on + go run bank.go case.go -u 'root@tcp(127.0.0.1:4000)/bank' -d 'root@tcp(127.0.0.1:3306)/bank' --test-round=20000 -a 127.0.0.1:20080 go: downloading go1.23.4 (linux/amd64) wait process cdc.test exit for 2-th time... wait process cdc.test exit for 2-th time... wait process cdc.test exit for 2-th time... + set +x ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 221 100 221 0 0 4700 0 --:--:-- --:--:-- --:--:-- 4702 100 221 100 221 0 0 4690 0 --:--:-- --:--:-- --:--:-- 4702 + synced_status='{"synced":true,"sink_checkpoint_ts":"2025-04-06 08:21:28.808","puller_resolved_ts":"1970-01-01 08:00:00.000","last_synced_ts":"1970-01-01 08:00:00.000","now_ts":"2025-04-06 08:21:35.000","info":"Data syncing is finished"}' ++ echo '{"synced":true,"sink_checkpoint_ts":"2025-04-06' '08:21:28.808","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2025-04-06' '08:21:35.000","info":"Data' syncing is 'finished"}' ++ jq .synced wait process cdc.test exit for 3-th time... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8302 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8302 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 06 Apr 2025 00:21:35 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/9059244d-df63-4544-be0d-9cc978a38a33 {UpstreamID:7489988629164751674 Namespace:default ID:9059244d-df63-4544-be0d-9cc978a38a33 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-06 08:21:26.135526297 +0800 CST StartTs:457152629583904770 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0037bb440 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457152629597274119} {CheckpointTs:457152631720378383 MinTableBarrierTs:457152631720378383 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/53acd7ff-c8e0-43bf-9961-a6e6448bec02 {"id":"53acd7ff-c8e0-43bf-9961-a6e6448bec02","address":"127.0.0.1:8302","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898892} /tidb/cdc/default/__cdc_meta__/capture/ddf0664d-5b64-470b-ad98-6496a2b72978 {"id":"ddf0664d-5b64-470b-ad98-6496a2b72978","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898882} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319608790502d6 ddf0664d-5b64-470b-ad98-6496a2b72978 /tidb/cdc/default/__cdc_meta__/owner/2231960879050371 53acd7ff-c8e0-43bf-9961-a6e6448bec02 /tidb/cdc/default/default/changefeed/info/9059244d-df63-4544-be0d-9cc978a38a33 {"upstream-id":7489988629164751674,"namespace":"default","changefeed-id":"9059244d-df63-4544-be0d-9cc978a38a33","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-06T08:21:26.135526297+08:00","start-ts":457152629583904770,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457152629597274119} /tidb/cdc/default/default/changefeed/status/9059244d-df63-4544-be0d-9cc978a38a33 {"checkpoint-ts":457152631720378383,"min-table-barrier-ts":457152631720378383,"admin-job-type":0} /tidb/cdc/default/default/task/position/53acd7ff-c8e0-43bf-9961-a6e6448bec02/9059244d-df63-4544-be0d-9cc978a38a33 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/ddf0664d-5b64-470b-ad98-6496a2b72978/9059244d-df63-4544-be0d-9cc978a38a33 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489988629164751674 {"id":7489988629164751674,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/9059244d-df63-4544-be0d-9cc978a38a33 {UpstreamID:7489988629164751674 Namespace:default ID:9059244d-df63-4544-be0d-9cc978a38a33 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-06 08:21:26.135526297 +0800 CST StartTs:457152629583904770 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0037bb440 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457152629597274119} {CheckpointTs:457152631720378383 MinTableBarrierTs:457152631720378383 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/53acd7ff-c8e0-43bf-9961-a6e6448bec02 {"id":"53acd7ff-c8e0-43bf-9961-a6e6448bec02","address":"127.0.0.1:8302","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898892} /tidb/cdc/default/__cdc_meta__/capture/ddf0664d-5b64-470b-ad98-6496a2b72978 {"id":"ddf0664d-5b64-470b-ad98-6496a2b72978","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898882} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319608790502d6 ddf0664d-5b64-470b-ad98-6496a2b72978 /tidb/cdc/default/__cdc_meta__/owner/2231960879050371 53acd7ff-c8e0-43bf-9961-a6e6448bec02 /tidb/cdc/default/default/changefeed/info/9059244d-df63-4544-be0d-9cc978a38a33 {"upstream-id":7489988629164751674,"namespace":"default","changefeed-id":"9059244d-df63-4544-be0d-9cc978a38a33","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-06T08:21:26.135526297+08:00","start-ts":457152629583904770,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457152629597274119} /tidb/cdc/default/default/changefeed/status/9059244d-df63-4544-be0d-9cc978a38a33 {"checkpoint-ts":457152631720378383,"min-table-barrier-ts":457152631720378383,"admin-job-type":0} /tidb/cdc/default/default/task/position/53acd7ff-c8e0-43bf-9961-a6e6448bec02/9059244d-df63-4544-be0d-9cc978a38a33 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/ddf0664d-5b64-470b-ad98-6496a2b72978/9059244d-df63-4544-be0d-9cc978a38a33 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} + grep -q 'failed to get info:' /tidb/cdc/default/default/upstream/7489988629164751674 {"id":7489988629164751674,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/9059244d-df63-4544-be0d-9cc978a38a33 {UpstreamID:7489988629164751674 Namespace:default ID:9059244d-df63-4544-be0d-9cc978a38a33 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-06 08:21:26.135526297 +0800 CST StartTs:457152629583904770 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0037bb440 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457152629597274119} {CheckpointTs:457152631720378383 MinTableBarrierTs:457152631720378383 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/53acd7ff-c8e0-43bf-9961-a6e6448bec02 {"id":"53acd7ff-c8e0-43bf-9961-a6e6448bec02","address":"127.0.0.1:8302","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898892} /tidb/cdc/default/__cdc_meta__/capture/ddf0664d-5b64-470b-ad98-6496a2b72978 {"id":"ddf0664d-5b64-470b-ad98-6496a2b72978","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898882} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319608790502d6 ddf0664d-5b64-470b-ad98-6496a2b72978 /tidb/cdc/default/__cdc_meta__/owner/2231960879050371 53acd7ff-c8e0-43bf-9961-a6e6448bec02 /tidb/cdc/default/default/changefeed/info/9059244d-df63-4544-be0d-9cc978a38a33 {"upstream-id":7489988629164751674,"namespace":"default","changefeed-id":"9059244d-df63-4544-be0d-9cc978a38a33","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-06T08:21:26.135526297+08:00","start-ts":457152629583904770,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457152629597274119} /tidb/cdc/default/default/changefeed/status/9059244d-df63-4544-be0d-9cc978a38a33 {"checkpoint-ts":457152631720378383,"min-table-barrier-ts":457152631720378383,"admin-job-type":0} /tidb/cdc/default/default/task/position/53acd7ff-c8e0-43bf-9961-a6e6448bec02/9059244d-df63-4544-be0d-9cc978a38a33 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/ddf0664d-5b64-470b-ad98-6496a2b72978/9059244d-df63-4544-be0d-9cc978a38a33 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489988629164751674 {"id":7489988629164751674,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x cdc cli capture list --server http://127.0.0.1:8302 |jq '.|length'|grep -E '^2$' 2 run task successfully % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 + status=true ++ echo '{"synced":true,"sink_checkpoint_ts":"2025-04-06' '08:21:28.808","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2025-04-06' '08:21:35.000","info":"Data' syncing is 'finished"}' ++ jq -r .sink_checkpoint_ts + sink_checkpoint_ts='2025-04-06 08:21:28.808' ++ echo '{"synced":true,"sink_checkpoint_ts":"2025-04-06' '08:21:28.808","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2025-04-06' '08:21:35.000","info":"Data' syncing is 'finished"}' ++ jq -r .puller_resolved_ts + puller_resolved_ts='1970-01-01 08:00:00.000' ++ echo '{"synced":true,"sink_checkpoint_ts":"2025-04-06' '08:21:28.808","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2025-04-06' '08:21:35.000","info":"Data' syncing is 'finished"}' ++ jq -r .last_synced_ts + last_synced_ts='1970-01-01 08:00:00.000' + '[' true '!=' true ']' + '[' '1970-01-01 08:00:00.000' '!=' '1970-01-01 08:00:00.000' ']' + '[' '1970-01-01 08:00:00.000' '!=' '1970-01-01 08:00:00.000' ']' ++ date '+%Y-%m-%d %H:%M:%S' + current='2025-04-06 08:21:35' + echo 'sink_checkpoint_ts is 2025-04-06' 08:21:28.808 sink_checkpoint_ts is 2025-04-06 08:21:28.808 ++ date -d '2025-04-06 08:21:28.808' +%s + checkpoint_timestamp=1743898888 ++ date -d '2025-04-06 08:21:35' +%s + current_timestamp=1743898895 + '[' 7 -gt 300 ']' + run_sql 'USE TEST;Create table t1(a int primary key, b int);insert into t1 values(1,2);insert into t1 values(2,3);' table new_ci_collation_test.t4 exists table new_ci_collation_test.t5 not exists for 1-th check, retry later wait process cdc.test exit for 3-th time... wait process cdc.test exit for 3-th time... process cdc.test already exit [Sun Apr 6 08:21:35 CST 2025] <<<<<< run test case tiflash success! >>>>>> + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 06 Apr 2025 00:21:35 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/e684662d-022f-4a79-9ef3-0a59dba96193 {UpstreamID:7489988628380945915 Namespace:default ID:e684662d-022f-4a79-9ef3-0a59dba96193 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-06 08:21:28.032207553 +0800 CST StartTs:457152628763131907 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003c81b00 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457152630100066306} {CheckpointTs:457152629169192984 MinTableBarrierTs:457152632000348167 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/38fd92d0-789a-4368-8d10-75b97b6cd5a0 {"id":"38fd92d0-789a-4368-8d10-75b97b6cd5a0","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898892} /tidb/cdc/default/__cdc_meta__/capture/afe53a47-596b-4147-92c4-b144550f6349 {"id":"afe53a47-596b-4147-92c4-b144550f6349","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898884} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319608790544fd afe53a47-596b-4147-92c4-b144550f6349 /tidb/cdc/default/__cdc_meta__/owner/2231960879054562 38fd92d0-789a-4368-8d10-75b97b6cd5a0 /tidb/cdc/default/default/changefeed/info/e684662d-022f-4a79-9ef3-0a59dba96193 {"upstream-id":7489988628380945915,"namespace":"default","changefeed-id":"e684662d-022f-4a79-9ef3-0a59dba96193","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-06T08:21:28.032207553+08:00","start-ts":457152628763131907,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457152630100066306} /tidb/cdc/default/default/changefeed/status/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":457152629169192984,"min-table-barrier-ts":457152632000348167,"admin-job-type":0} /tidb/cdc/default/default/task/position/38fd92d0-789a-4368-8d10-75b97b6cd5a0/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/afe53a47-596b-4147-92c4-b144550f6349/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489988628380945915 {"id":7489988628380945915,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/e684662d-022f-4a79-9ef3-0a59dba96193 {UpstreamID:7489988628380945915 Namespace:default ID:e684662d-022f-4a79-9ef3-0a59dba96193 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-06 08:21:28.032207553 +0800 CST StartTs:457152628763131907 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003c81b00 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457152630100066306} {CheckpointTs:457152629169192984 MinTableBarrierTs:457152632000348167 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/38fd92d0-789a-4368-8d10-75b97b6cd5a0 {"id":"38fd92d0-789a-4368-8d10-75b97b6cd5a0","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898892} /tidb/cdc/default/__cdc_meta__/capture/afe53a47-596b-4147-92c4-b144550f6349 {"id":"afe53a47-596b-4147-92c4-b144550f6349","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898884} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319608790544fd afe53a47-596b-4147-92c4-b144550f6349 /tidb/cdc/default/__cdc_meta__/owner/2231960879054562 38fd92d0-789a-4368-8d10-75b97b6cd5a0 /tidb/cdc/default/default/changefeed/info/e684662d-022f-4a79-9ef3-0a59dba96193 {"upstream-id":7489988628380945915,"namespace":"default","changefeed-id":"e684662d-022f-4a79-9ef3-0a59dba96193","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-06T08:21:28.032207553+08:00","start-ts":457152628763131907,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457152630100066306} /tidb/cdc/default/default/changefeed/status/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":457152629169192984,"min-table-barrier-ts":457152632000348167,"admin-job-type":0} /tidb/cdc/default/default/task/position/38fd92d0-789a-4368-8d10-75b97b6cd5a0/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/afe53a47-596b-4147-92c4-b144550f6349/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489988628380945915 {"id":7489988628380945915,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: changefeedID: default/e684662d-022f-4a79-9ef3-0a59dba96193 {UpstreamID:7489988628380945915 Namespace:default ID:e684662d-022f-4a79-9ef3-0a59dba96193 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-06 08:21:28.032207553 +0800 CST StartTs:457152628763131907 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003c81b00 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457152630100066306} {CheckpointTs:457152629169192984 MinTableBarrierTs:457152632000348167 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/38fd92d0-789a-4368-8d10-75b97b6cd5a0 {"id":"38fd92d0-789a-4368-8d10-75b97b6cd5a0","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898892} /tidb/cdc/default/__cdc_meta__/capture/afe53a47-596b-4147-92c4-b144550f6349 {"id":"afe53a47-596b-4147-92c4-b144550f6349","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898884} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319608790544fd afe53a47-596b-4147-92c4-b144550f6349 /tidb/cdc/default/__cdc_meta__/owner/2231960879054562 38fd92d0-789a-4368-8d10-75b97b6cd5a0 /tidb/cdc/default/default/changefeed/info/e684662d-022f-4a79-9ef3-0a59dba96193 {"upstream-id":7489988628380945915,"namespace":"default","changefeed-id":"e684662d-022f-4a79-9ef3-0a59dba96193","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-06T08:21:28.032207553+08:00","start-ts":457152628763131907,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457152630100066306} /tidb/cdc/default/default/changefeed/status/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":457152629169192984,"min-table-barrier-ts":457152632000348167,"admin-job-type":0} /tidb/cdc/default/default/task/position/38fd92d0-789a-4368-8d10-75b97b6cd5a0/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/afe53a47-596b-4147-92c4-b144550f6349/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489988628380945915 {"id":7489988628380945915,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x cdc.test cli capture list 2>&1 | grep -v "afe53a47-596b-4147-92c4-b144550f6349" | grep -v cluster_id | grep id \033[0;36m<<< Run all test success >>>\033[0m cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Sun Apr 6 08:21:35 CST 2025] <<<<<< run test case simple success! >>>>>> check_changefeed_state http://127.0.0.1:2379 changefeed-initialize-error normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-initialize-error + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-initialize-error -s + info='{ "upstream_id": 7489988418452591948, "namespace": "default", "id": "changefeed-initialize-error", "state": "normal", "checkpoint_tso": 457152631970463745, "checkpoint_time": "2025-04-06 08:21:35.151", "error": null }' + echo '{ "upstream_id": 7489988418452591948, "namespace": "default", "id": "changefeed-initialize-error", "state": "normal", "checkpoint_tso": 457152631970463745, "checkpoint_time": "2025-04-06 08:21:35.151", "error": null }' { "upstream_id": 7489988418452591948, "namespace": "default", "id": "changefeed-initialize-error", "state": "normal", "checkpoint_tso": 457152631970463745, "checkpoint_time": "2025-04-06 08:21:35.151", "error": null } ++ echo '{' '"upstream_id":' 7489988418452591948, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"normal",' '"checkpoint_tso":' 457152631970463745, '"checkpoint_time":' '"2025-04-06' '08:21:35.151",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7489988418452591948, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"normal",' '"checkpoint_tso":' 457152631970463745, '"checkpoint_time":' '"2025-04-06' '08:21:35.151",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.6584.out cli changefeed pause -c changefeed-initialize-error table ddl_manager.finish_mark not exists for 36-th check, retry later + check_table_exists test.t1 127.0.0.1 3306 table test.t1 not exists for 1-th check, retry later cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Sun Apr 6 08:21:35 CST 2025] <<<<<< run test case http_api_tls success! >>>>>> go: downloading github.com/go-sql-driver/mysql v1.6.0 go: downloading github.com/sirupsen/logrus v1.8.1 go: downloading github.com/pingcap/errors v0.11.5-0.20211009033009-93128226aaa3 go: downloading github.com/pingcap/tidb/parser v0.0.0-20211122065349-4d16aabf900b go: downloading go.uber.org/atomic v1.9.0 go: downloading golang.org/x/sys v0.0.0-20211117180635-dee7805ff2e1 go: downloading golang.org/x/text v0.3.7 go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading go.uber.org/zap v1.19.1 go: downloading github.com/pingcap/log v0.0.0-20210906054005-afc726e70354 go: downloading gopkg.in/natefinch/lumberjack.v2 v2.0.0 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 go: downloading go.uber.org/multierr v1.7.0 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... "cluster-id": "default" "id": "38fd92d0-789a-4368-8d10-75b97b6cd5a0", "cluster-id": "default" run task successfully table new_ci_collation_test.t5 exists check diff failed 1-th time, retry later run task successfully check_changefeed_state http://127.0.0.1:2379 1ca16915-453d-4d6c-ae3b-5cef0432008b normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=1ca16915-453d-4d6c-ae3b-5cef0432008b + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c 1ca16915-453d-4d6c-ae3b-5cef0432008b -s + set +x check_changefeed_state http://127.0.0.1:2379 changefeed-initialize-error stopped null + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-initialize-error + expected_state=stopped + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-initialize-error -s table test.t1 exists + sleep 5 + info='{ "upstream_id": 7489988635534264163, "namespace": "default", "id": "1ca16915-453d-4d6c-ae3b-5cef0432008b", "state": "normal", "checkpoint_tso": 457152631578034177, "checkpoint_time": "2025-04-06 08:21:33.654", "error": null }' + echo '{ "upstream_id": 7489988635534264163, "namespace": "default", "id": "1ca16915-453d-4d6c-ae3b-5cef0432008b", "state": "normal", "checkpoint_tso": 457152631578034177, "checkpoint_time": "2025-04-06 08:21:33.654", "error": null }' { "upstream_id": 7489988635534264163, "namespace": "default", "id": "1ca16915-453d-4d6c-ae3b-5cef0432008b", "state": "normal", "checkpoint_tso": 457152631578034177, "checkpoint_time": "2025-04-06 08:21:33.654", "error": null } ++ echo '{' '"upstream_id":' 7489988635534264163, '"namespace":' '"default",' '"id":' '"1ca16915-453d-4d6c-ae3b-5cef0432008b",' '"state":' '"normal",' '"checkpoint_tso":' 457152631578034177, '"checkpoint_time":' '"2025-04-06' '08:21:33.654",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7489988635534264163, '"namespace":' '"default",' '"id":' '"1ca16915-453d-4d6c-ae3b-5cef0432008b",' '"state":' '"normal",' '"checkpoint_tso":' 457152631578034177, '"checkpoint_time":' '"2025-04-06' '08:21:33.654",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully check_safepoint_forward http://127.0.0.1:2379 7489988635534264163 457152631578034176 457152631578034177 + info='{ "upstream_id": 7489988418452591948, "namespace": "default", "id": "changefeed-initialize-error", "state": "stopped", "checkpoint_tso": 457152631970463745, "checkpoint_time": "2025-04-06 08:21:35.151", "error": null }' + echo '{ "upstream_id": 7489988418452591948, "namespace": "default", "id": "changefeed-initialize-error", "state": "stopped", "checkpoint_tso": 457152631970463745, "checkpoint_time": "2025-04-06 08:21:35.151", "error": null }' { "upstream_id": 7489988418452591948, "namespace": "default", "id": "changefeed-initialize-error", "state": "stopped", "checkpoint_tso": 457152631970463745, "checkpoint_time": "2025-04-06 08:21:35.151", "error": null } ++ echo '{' '"upstream_id":' 7489988418452591948, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"stopped",' '"checkpoint_tso":' 457152631970463745, '"checkpoint_time":' '"2025-04-06' '08:21:35.151",' '"error":' null '}' ++ jq -r .state + state=stopped + [[ ! stopped == \s\t\o\p\p\e\d ]] ++ echo '{' '"upstream_id":' 7489988418452591948, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"stopped",' '"checkpoint_tso":' 457152631970463745, '"checkpoint_time":' '"2025-04-06' '08:21:35.151",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.6647.out cli changefeed resume -c changefeed-initialize-error capture_id: 38fd92d0-789a-4368-8d10-75b97b6cd5a0 table ddl_manager.finish_mark not exists for 37-th check, retry later PASS coverage: 2.1% of statements in github.com/pingcap/tiflow/... check diff failed 1-th time, retry later go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20241223052309-3735ed55a394 go: downloading github.com/pingcap/kvproto v0.0.0-20241120022153-92b0414aeed8 go: downloading github.com/pingcap/errors v0.11.5-0.20240318064555-6bd07397691f go: downloading github.com/pingcap/log v1.1.1-0.20241212030209-7e3ff8601a2a go: downloading github.com/tikv/client-go/v2 v2.0.8-0.20241209094930-06d7f4b9233b go: downloading github.com/tikv/pd/client v0.0.0-20241111073742-238d4d79ea31 go: downloading github.com/BurntSushi/toml v1.4.1-0.20240526193622-a339e1f7089c go: downloading go.uber.org/zap v1.27.0 go: downloading github.com/pingcap/failpoint v0.0.0-20240528011301-b51a646c7c86 go: downloading gopkg.in/natefinch/lumberjack.v2 v2.2.1 go: downloading go.uber.org/atomic v1.11.0 go: downloading github.com/tikv/pd v1.1.0-beta.0.20240407022249-7179657d129b go: downloading go.uber.org/multierr v1.11.0 go: downloading github.com/golang/protobuf v1.5.4 go: downloading github.com/gogo/protobuf v1.3.2 go: downloading google.golang.org/grpc v1.64.0 go: downloading github.com/pkg/errors v0.9.1 go: downloading github.com/opentracing/opentracing-go v1.2.0 go: downloading github.com/google/uuid v1.6.0 go: downloading github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a go: downloading github.com/prometheus/client_golang v1.20.5 go: downloading go.etcd.io/etcd/api/v3 v3.5.12 go: downloading go.etcd.io/etcd/client/v3 v3.5.12 go: downloading golang.org/x/sync v0.10.0 go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 go: downloading github.com/prometheus/client_model v0.6.1 go: downloading github.com/docker/go-units v0.5.0 go: downloading github.com/dgryski/go-farm v0.0.0-20240924180020-3414d57e47da go: downloading github.com/twmb/murmur3 v1.1.6 go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading github.com/google/btree v1.1.2 go: downloading github.com/pingcap/log v1.1.1-0.20241212030209-7e3ff8601a2a go: downloading github.com/pingcap/errors v0.11.5-0.20240318064555-6bd07397691f go: downloading github.com/BurntSushi/toml v1.4.1-0.20240526193622-a339e1f7089c go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20241223052309-3735ed55a394 go: downloading go.uber.org/zap v1.27.0 go: downloading github.com/pingcap/failpoint v0.0.0-20240528011301-b51a646c7c86 go: downloading gopkg.in/natefinch/lumberjack.v2 v2.2.1 go: downloading go.uber.org/atomic v1.11.0 go: downloading go.uber.org/multierr v1.11.0 wait process 2744 exit for 1-th time... run task successfully go: downloading google.golang.org/protobuf v1.35.2 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.12 go: downloading google.golang.org/genproto/googleapis/api v0.0.0-20240401170217-c3f982113cda go: downloading github.com/coreos/go-semver v0.3.1 go: downloading github.com/coreos/go-systemd/v22 v22.5.0 go: downloading github.com/prometheus/common v0.61.0 go: downloading github.com/beorn7/perks v1.0.1 go: downloading github.com/prometheus/procfs v0.15.1 go: downloading github.com/cespare/xxhash/v2 v2.3.0 go: downloading google.golang.org/genproto v0.0.0-20240401170217-c3f982113cda go: downloading github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 check_changefeed_state http://127.0.0.1:2379 1ca16915-453d-4d6c-ae3b-5cef0432008b stopped null + endpoints=http://127.0.0.1:2379 + changefeed_id=1ca16915-453d-4d6c-ae3b-5cef0432008b + expected_state=stopped + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c 1ca16915-453d-4d6c-ae3b-5cef0432008b -s go: downloading golang.org/x/sys v0.28.0 go: downloading golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 go: downloading github.com/cloudfoundry/gosigar v1.3.6 wait process 2744 exit for 2-th time... + info='{ "upstream_id": 7489988635534264163, "namespace": "default", "id": "1ca16915-453d-4d6c-ae3b-5cef0432008b", "state": "stopped", "checkpoint_tso": 457152631578034177, "checkpoint_time": "2025-04-06 08:21:33.654", "error": null }' + echo '{ "upstream_id": 7489988635534264163, "namespace": "default", "id": "1ca16915-453d-4d6c-ae3b-5cef0432008b", "state": "stopped", "checkpoint_tso": 457152631578034177, "checkpoint_time": "2025-04-06 08:21:33.654", "error": null }' { "upstream_id": 7489988635534264163, "namespace": "default", "id": "1ca16915-453d-4d6c-ae3b-5cef0432008b", "state": "stopped", "checkpoint_tso": 457152631578034177, "checkpoint_time": "2025-04-06 08:21:33.654", "error": null } ++ echo '{' '"upstream_id":' 7489988635534264163, '"namespace":' '"default",' '"id":' '"1ca16915-453d-4d6c-ae3b-5cef0432008b",' '"state":' '"stopped",' '"checkpoint_tso":' 457152631578034177, '"checkpoint_time":' '"2025-04-06' '08:21:33.654",' '"error":' null '}' ++ jq -r .state + state=stopped + [[ ! stopped == \s\t\o\p\p\e\d ]] ++ echo '{' '"upstream_id":' 7489988635534264163, '"namespace":' '"default",' '"id":' '"1ca16915-453d-4d6c-ae3b-5cef0432008b",' '"state":' '"stopped",' '"checkpoint_tso":' 457152631578034177, '"checkpoint_time":' '"2025-04-06' '08:21:33.654",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully go: downloading google.golang.org/genproto/googleapis/rpc v0.0.0-20240515191416-fc5f0ca64291 go: downloading golang.org/x/net v0.33.0 check_changefeed_state http://127.0.0.1:2379 892434ed-01b1-4a02-b81f-d1daca93ac09 normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=892434ed-01b1-4a02-b81f-d1daca93ac09 + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c 892434ed-01b1-4a02-b81f-d1daca93ac09 -s + info='{ "upstream_id": 7489988635534264163, "namespace": "default", "id": "892434ed-01b1-4a02-b81f-d1daca93ac09", "state": "normal", "checkpoint_tso": 457152633098207233, "checkpoint_time": "2025-04-06 08:21:39.453", "error": null }' + echo '{ "upstream_id": 7489988635534264163, "namespace": "default", "id": "892434ed-01b1-4a02-b81f-d1daca93ac09", "state": "normal", "checkpoint_tso": 457152633098207233, "checkpoint_time": "2025-04-06 08:21:39.453", "error": null }' { "upstream_id": 7489988635534264163, "namespace": "default", "id": "892434ed-01b1-4a02-b81f-d1daca93ac09", "state": "normal", "checkpoint_tso": 457152633098207233, "checkpoint_time": "2025-04-06 08:21:39.453", "error": null } ++ echo '{' '"upstream_id":' 7489988635534264163, '"namespace":' '"default",' '"id":' '"892434ed-01b1-4a02-b81f-d1daca93ac09",' '"state":' '"normal",' '"checkpoint_tso":' 457152633098207233, '"checkpoint_time":' '"2025-04-06' '08:21:39.453",' '"error":' null '}' ++ jq -r .state go: downloading golang.org/x/text v0.21.0 check diff failed 2-th time, retry later + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7489988635534264163, '"namespace":' '"default",' '"id":' '"892434ed-01b1-4a02-b81f-d1daca93ac09",' '"state":' '"normal",' '"checkpoint_tso":' 457152633098207233, '"checkpoint_time":' '"2025-04-06' '08:21:39.453",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully check_safepoint_equal http://127.0.0.1:2379 7489988635534264163 go: downloading github.com/pingcap/errors v0.11.5-0.20240318064555-6bd07397691f go: downloading github.com/spf13/cobra v1.8.1 go: downloading go.uber.org/zap v1.27.0 go: downloading github.com/pingcap/log v1.1.1-0.20241212030209-7e3ff8601a2a go: downloading github.com/go-sql-driver/mysql v1.7.1 go: downloading golang.org/x/sync v0.10.0 + set +x check_changefeed_state http://127.0.0.1:2379 changefeed-initialize-error normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-initialize-error + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-initialize-error -s /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (2744) - No such process wait process 2744 exit for 3-th time... process 2744 already exit cdc.test cli capture list --server 'http://127.0.0.1:8301' 2>&1 |grep 38fd92d0-789a-4368-8d10-75b97b6cd5a0 -A1 | grep '"is-owner": true' go: downloading google.golang.org/grpc v1.64.0 go: downloading gopkg.in/natefinch/lumberjack.v2 v2.2.1 go: downloading go.uber.org/atomic v1.11.0 go: downloading github.com/spf13/pflag v1.0.5 go: downloading go.uber.org/multierr v1.11.0 + info='{ "upstream_id": 7489988418452591948, "namespace": "default", "id": "changefeed-initialize-error", "state": "normal", "checkpoint_tso": 457152631970463745, "checkpoint_time": "2025-04-06 08:21:35.151", "error": null }' + echo '{ "upstream_id": 7489988418452591948, "namespace": "default", "id": "changefeed-initialize-error", "state": "normal", "checkpoint_tso": 457152631970463745, "checkpoint_time": "2025-04-06 08:21:35.151", "error": null }' { "upstream_id": 7489988418452591948, "namespace": "default", "id": "changefeed-initialize-error", "state": "normal", "checkpoint_tso": 457152631970463745, "checkpoint_time": "2025-04-06 08:21:35.151", "error": null } ++ echo '{' '"upstream_id":' 7489988418452591948, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"normal",' '"checkpoint_tso":' 457152631970463745, '"checkpoint_time":' '"2025-04-06' '08:21:35.151",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7489988418452591948, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"normal",' '"checkpoint_tso":' 457152631970463745, '"checkpoint_time":' '"2025-04-06' '08:21:35.151",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.6726.out cli changefeed remove -c changefeed-initialize-error table ddl_manager.finish_mark not exists for 38-th check, retry later go: downloading golang.org/x/sys v0.28.0 go: downloading google.golang.org/protobuf v1.35.2 go: downloading google.golang.org/genproto/googleapis/rpc v0.0.0-20240515191416-fc5f0ca64291 go: downloading google.golang.org/genproto v0.0.0-20240401170217-c3f982113cda check diff failed 2-th time, retry later Changefeed remove successfully. ID: changefeed-initialize-error CheckpointTs: 457152633281183746 SinkURI: mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... go: downloading github.com/go-sql-driver/mysql v1.7.1 go: downloading github.com/pingcap/tidb/pkg/parser v0.0.0-20241223052309-3735ed55a394 go: downloading github.com/json-iterator/go v1.1.12 go: downloading github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 go: downloading golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 go: downloading github.com/golang/protobuf v1.5.4 go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 go: downloading github.com/danjacques/gofslock v0.0.0-20240212154529-d899e02bfe22 go: downloading github.com/tikv/pd/client v0.0.0-20241111073742-238d4d79ea31 go: downloading github.com/tikv/client-go/v2 v2.0.8-0.20241209094930-06d7f4b9233b go: downloading golang.org/x/text v0.21.0 go: downloading github.com/prometheus/client_golang v1.20.5 go: downloading github.com/prometheus/client_model v0.6.1 go: downloading github.com/pingcap/kvproto v0.0.0-20241120022153-92b0414aeed8 go: downloading github.com/docker/go-units v0.5.0 go: downloading golang.org/x/sync v0.10.0 go: downloading github.com/opentracing/opentracing-go v1.2.0 go: downloading golang.org/x/net v0.33.0 go: downloading golang.org/x/tools v0.28.0 go: downloading github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a go: downloading go.etcd.io/etcd/client/v3 v3.5.12 go: downloading github.com/pingcap/tipb v0.0.0-20241105053214-f91fdb81a69e go: downloading github.com/influxdata/tdigest v0.0.1 go: downloading github.com/spf13/pflag v1.0.5 go: downloading github.com/shirou/gopsutil/v3 v3.24.5 go: downloading github.com/pingcap/sysutil v1.0.1-0.20240311050922-ae81ee01f3a5 go: downloading google.golang.org/grpc v1.64.0 go: downloading github.com/coocood/freecache v1.2.1 go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible go: downloading github.com/yangkeao/ldap/v3 v3.4.5-0.20230421065457-369a3bab1117 go: downloading github.com/tidwall/btree v1.7.0 go: downloading github.com/twmb/murmur3 v1.1.6 go: downloading github.com/gorilla/mux v1.8.1 go: downloading github.com/stretchr/testify v1.10.0 go: downloading github.com/opentracing/basictracer-go v1.1.0 go: downloading github.com/coreos/go-semver v0.3.1 go: downloading github.com/google/uuid v1.6.0 go: downloading github.com/bits-and-blooms/bitset v1.14.3 go: downloading gopkg.in/yaml.v2 v2.4.0 go: downloading github.com/google/btree v1.1.2 go: downloading github.com/otiai10/copy v1.2.0 go: downloading github.com/golang/snappy v0.0.4 go: downloading golang.org/x/sys v0.28.0 go: downloading cloud.google.com/go/storage v1.39.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v1.16.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 go: downloading github.com/aws/aws-sdk-go v1.55.5 go: downloading github.com/tikv/pd v1.1.0-beta.0.20240407022249-7179657d129b go: downloading github.com/go-resty/resty/v2 v2.11.0 "is-owner": true, run task successfully test_kill_owner: pass Cache restored successfully (ws/jenkins-pingcap-tiflow-periodics_cdc_integration_mysql_test-234/tiflow-cdc) 4028702720 bytes in 9.02 secs (446869729 bytes/sec) [Pipeline] { [Pipeline] timeout Timeout set to expire in 45 min [Pipeline] { go: downloading github.com/klauspost/compress v1.17.9 go: downloading github.com/ks3sdklib/aws-sdk-go v1.2.9 go: downloading cloud.google.com/go v0.112.2 go: downloading golang.org/x/oauth2 v0.24.0 go: downloading google.golang.org/api v0.170.0 [Pipeline] sh go: downloading github.com/coocood/freecache v1.2.1 go: downloading github.com/pingcap/tidb/pkg/parser v0.0.0-20241223052309-3735ed55a394 go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible go: downloading github.com/spf13/pflag v1.0.5 go: downloading github.com/go-sql-driver/mysql v1.7.1 go: downloading github.com/opentracing/basictracer-go v1.1.0 go: downloading github.com/pingcap/tipb v0.0.0-20241105053214-f91fdb81a69e go: downloading github.com/json-iterator/go v1.1.12 go: downloading github.com/dgraph-io/ristretto v0.1.1 go: downloading github.com/pingcap/sysutil v1.0.1-0.20240311050922-ae81ee01f3a5 go: downloading gopkg.in/yaml.v2 v2.4.0 go: downloading github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 go: downloading github.com/danjacques/gofslock v0.0.0-20240212154529-d899e02bfe22 go: downloading github.com/influxdata/tdigest v0.0.1 go: downloading github.com/gorilla/mux v1.8.1 go: downloading github.com/shirou/gopsutil/v3 v3.24.5 go: downloading github.com/bits-and-blooms/bitset v1.14.3 go: downloading cloud.google.com/go/storage v1.39.1 go: downloading github.com/dolthub/swiss v0.2.1 go: downloading github.com/yangkeao/ldap/v3 v3.4.5-0.20230421065457-369a3bab1117 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v1.16.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0 go: downloading github.com/stretchr/testify v1.10.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 go: downloading github.com/tidwall/btree v1.7.0 go: downloading golang.org/x/tools v0.28.0 go: downloading github.com/qri-io/jsonschema v0.2.1 go: downloading github.com/otiai10/copy v1.2.0 go: downloading go.uber.org/mock v0.4.0 go: downloading github.com/golang/snappy v0.0.4 go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 go: downloading github.com/aws/aws-sdk-go v1.55.5 go: downloading github.com/go-resty/resty/v2 v2.11.0 go: downloading github.com/klauspost/compress v1.17.9 go: downloading github.com/ks3sdklib/aws-sdk-go v1.2.9 go: downloading golang.org/x/oauth2 v0.24.0 go: downloading google.golang.org/api v0.170.0 go: downloading github.com/cockroachdb/pebble v1.1.0 go: downloading golang.org/x/time v0.7.0 go: downloading github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df check diff failed 3-th time, retry later go: downloading go.etcd.io/etcd/api/v3 v3.5.12 go: downloading go.uber.org/mock v0.4.0 go: downloading github.com/cockroachdb/pebble v1.1.0 go: downloading github.com/jfcg/sorty/v2 v2.1.0 go: downloading github.com/carlmjohnson/flagext v0.21.0 go: downloading github.com/gogo/protobuf v1.3.2 go: downloading github.com/qri-io/jsonschema v0.2.1 go: downloading github.com/dolthub/swiss v0.2.1 go: downloading golang.org/x/time v0.7.0 go: downloading google.golang.org/protobuf v1.35.2 go: downloading github.com/pkg/errors v0.9.1 go: downloading github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef go: downloading github.com/cespare/xxhash/v2 v2.3.0 go: downloading github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 go: downloading github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 go: downloading github.com/go-asn1-ber/asn1-ber v1.5.4 go: downloading github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc go: downloading github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.12 go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 go: downloading github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 go: downloading golang.org/x/crypto v0.31.0 go: downloading github.com/beorn7/perks v1.0.1 go: downloading github.com/prometheus/common v0.61.0 go: downloading github.com/prometheus/procfs v0.15.1 go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible go: downloading github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df go: downloading github.com/dgraph-io/ristretto v0.1.1 go: downloading github.com/cloudfoundry/gosigar v1.3.6 go: downloading github.com/spkg/bom v1.0.0 wait process cdc.test exit for 1-th time... go: downloading github.com/jfcg/sorty/v2 v2.1.0 go: downloading github.com/carlmjohnson/flagext v0.21.0 go: downloading github.com/spkg/bom v1.0.0 go: downloading github.com/xitongsys/parquet-go v1.6.3-0.20240520233950-75e935fc3e17 go: downloading github.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0 go: downloading github.com/lestrrat-go/jwx/v2 v2.0.21 go: downloading github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 go: downloading cloud.google.com/go v0.112.2 go: downloading github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef go: downloading github.com/dolthub/maphash v0.1.0 go: downloading github.com/jellydator/ttlcache/v3 v3.0.1 go: downloading github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 go: downloading github.com/go-asn1-ber/asn1-ber v1.5.4 go: downloading github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc go: downloading github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 go: downloading github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 go: downloading golang.org/x/crypto v0.31.0 go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading github.com/jedib0t/go-pretty/v6 v6.2.2 go: downloading github.com/cheggaaa/pb/v3 v3.0.8 go: downloading golang.org/x/term v0.27.0 go: downloading github.com/qri-io/jsonpointer v0.1.1 go: downloading github.com/fatih/color v1.18.0 go: downloading github.com/vbauerster/mpb/v7 v7.5.3 go: downloading github.com/spf13/cobra v1.8.1 go: downloading github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 go: downloading github.com/robfig/cron/v3 v3.0.1 go: downloading github.com/google/pprof v0.0.0-20241001023024-f4c0cfd0cf1d go: downloading github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f go: downloading cloud.google.com/go/compute/metadata v0.3.0 go: downloading github.com/pingcap/badger v1.5.1-0.20241015064302-38533b6cbf8d go: downloading github.com/jfcg/sixb v1.3.8 go: downloading github.com/robfig/cron v1.2.0 go: downloading github.com/tklauser/go-sysconf v0.3.12 go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible go: downloading github.com/xitongsys/parquet-go v1.6.3-0.20240520233950-75e935fc3e17 go: downloading github.com/google/pprof v0.0.0-20241001023024-f4c0cfd0cf1d go: downloading github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f go: downloading github.com/dgryski/go-farm v0.0.0-20240924180020-3414d57e47da go: downloading github.com/jedib0t/go-pretty/v6 v6.2.2 + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh mysql G19 go: downloading github.com/apache/thrift v0.16.0 go: downloading github.com/dustin/go-humanize v1.0.1 go: downloading github.com/golang/glog v1.2.0 go: downloading github.com/Masterminds/semver v1.5.0 go: downloading k8s.io/api v0.29.11 go: downloading github.com/emirpasic/gods v1.18.1 go: downloading github.com/VividCortex/ewma v1.2.0 go: downloading github.com/mattn/go-colorable v0.1.13 go: downloading github.com/mattn/go-runewidth v0.0.16 go: downloading github.com/mattn/go-isatty v0.0.20 go: downloading cloud.google.com/go/iam v1.1.7 go: downloading github.com/googleapis/gax-go/v2 v2.12.3 go: downloading github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d go: downloading github.com/apache/arrow/go/v12 v12.0.1 go: downloading github.com/goccy/go-reflect v1.2.0 go: downloading github.com/tklauser/numcpus v0.6.1 go: downloading github.com/kylelemons/godebug v1.1.0 go: downloading github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c go: downloading github.com/lestrrat-go/blackmagic v1.0.2 go: downloading github.com/lestrrat-go/iter v1.0.2 go: downloading github.com/lestrrat-go/httprc v1.0.5 go: downloading github.com/lestrrat-go/option v1.0.1 go: downloading go.opencensus.io v0.23.1-0.20220331163232-052120675fac go: downloading go.opentelemetry.io/otel v1.24.0 go: downloading go.opentelemetry.io/otel/trace v1.24.0 go: downloading github.com/pierrec/lz4/v4 v4.1.18 go: downloading github.com/rivo/uniseg v0.4.7 go: downloading github.com/golang-jwt/jwt/v5 v5.2.1 go: downloading github.com/golang-jwt/jwt v3.2.2+incompatible go: downloading cloud.google.com/go/kms v1.15.8 table ddl_manager.finish_mark not exists for 39-th check, retry later go: downloading github.com/jellydator/ttlcache/v3 v3.0.1 go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading github.com/lestrrat-go/jwx/v2 v2.0.21 go: downloading github.com/pingcap/badger v1.5.1-0.20241015064302-38533b6cbf8d go: downloading github.com/tklauser/go-sysconf v0.3.12 Run cases: changefeed_fast_fail batch_update_to_no_batch changefeed_resume_with_checkpoint_ts PROW_JOB_ID=1ff7fcb7-127a-11f0-bebb-96e16cfa81f4 JENKINS_NODE_COOKIE=a07d8ad2-d3df-4aaa-90cf-119b0d4d4d3d BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/ GOLANG_VERSION=1.23.3 HOSTNAME=pingcap-tiflow-periodics-cdc-integration-mysql-test-234-4-x87nv HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-periodics_cdc_integration_mysql_test-234 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 JOB_SPEC={"type":"periodic","job":"pingcap/tiflow/periodics_cdc_integration_mysql_test","buildid":"1908671202193313792","prowjobid":"1ff7fcb7-127a-11f0-bebb-96e16cfa81f4","extra_refs":[{"org":"pingcap","repo":"tiflow","base_ref":"master"}]} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ JOB_NAME=pingcap/tiflow/periodics_cdc_integration_mysql_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#234 TEST_GROUP=G19 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1908671202193313792 GOLANG_DOWNLOAD_SHA256=a0afb9744c00648bafb1b90b4aba5bdb86f424f02f9275399ce0c20b93a2c3a8 JOB_BASE_NAME=periodics_cdc_integration_mysql_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=b2a59508c11c86ba701c4af25fc536a9475b82c6 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_periodics_cdc_integration_mysql_test_234-44vnf GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-8f9954e717ae06be5a19c339deaa805ddd7d685252d85c8682580d893f46201d NODE_LABELS=pingcap_tiflow_periodics_cdc_integration_mysql_test_234-44vnf pingcap-tiflow-periodics-cdc-integration-mysql-test-234-4-x87nv GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-periodics-cdc-integration-mysql-test-234-4-x87nv GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/display/redirect BUILD_NUMBER=234 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.23.3.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/changefeed_fast_fail/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... go: downloading github.com/lestrrat-go/httpcc v1.0.1 go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da go: downloading github.com/ncw/directio v1.0.5 go: downloading github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 go: downloading github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 go: downloading github.com/klauspost/cpuid v1.3.1 wait process cdc.test exit for 2-th time... + set +x go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading cloud.google.com/go/compute/metadata v0.3.0 go: downloading github.com/dolthub/maphash v0.1.0 go: downloading github.com/jfcg/sixb v1.3.8 go: downloading google.golang.org/genproto/googleapis/api v0.0.0-20240401170217-c3f982113cda go: downloading cloud.google.com/go/iam v1.1.7 go: downloading github.com/googleapis/gax-go/v2 v2.12.3 go: downloading google.golang.org/genproto v0.0.0-20240401170217-c3f982113cda go: downloading go.opentelemetry.io/otel/metric v1.24.0 go: downloading github.com/go-logr/logr v1.4.1 go: downloading github.com/go-logr/stdr v1.2.2 go: downloading github.com/coreos/go-systemd/v22 v22.5.0 go: downloading github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 go: downloading github.com/qri-io/jsonpointer v0.1.1 go: downloading github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 go: downloading github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c go: downloading github.com/cheggaaa/pb/v3 v3.0.8 go: downloading golang.org/x/term v0.27.0 go: downloading github.com/kylelemons/godebug v1.1.0 go: downloading github.com/apache/thrift v0.16.0 go: downloading github.com/robfig/cron/v3 v3.0.1 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec go: downloading github.com/mattn/go-runewidth v0.0.16 go: downloading github.com/tklauser/numcpus v0.6.1 go: downloading github.com/lestrrat-go/blackmagic v1.0.2 go: downloading github.com/lestrrat-go/httprc v1.0.5 go: downloading github.com/lestrrat-go/iter v1.0.2 go: downloading github.com/lestrrat-go/option v1.0.1 go: downloading go.opencensus.io v0.23.1-0.20220331163232-052120675fac go: downloading go.opentelemetry.io/otel v1.24.0 go: downloading github.com/robfig/cron v1.2.0 go: downloading go.opentelemetry.io/otel/trace v1.24.0 go: downloading google.golang.org/genproto/googleapis/rpc v0.0.0-20240515191416-fc5f0ca64291 go: downloading github.com/golang-jwt/jwt/v5 v5.2.1 go: downloading github.com/dustin/go-humanize v1.0.1 go: downloading github.com/golang/glog v1.2.0 go: downloading github.com/apache/arrow/go/v12 v12.0.1 go: downloading github.com/goccy/go-reflect v1.2.0 go: downloading github.com/VividCortex/ewma v1.2.0 go: downloading github.com/fatih/color v1.18.0 go: downloading github.com/mattn/go-colorable v0.1.13 go: downloading github.com/mattn/go-isatty v0.0.20 go: downloading github.com/ncw/directio v1.0.5 go: downloading github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 go: downloading github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 go: downloading github.com/rivo/uniseg v0.4.7 go: downloading github.com/lestrrat-go/httpcc v1.0.1 go: downloading github.com/golang-jwt/jwt v3.2.2+incompatible go: downloading github.com/pierrec/lz4/v4 v4.1.18 go: downloading github.com/klauspost/cpuid v1.3.1 cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit run test case test_hang_up_owner [Sun Apr 6 08:21:42 CST 2025] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.30793081.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_hang_up_owner.server1.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_hang_up_owner.server1 --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 wait process cdc.test exit for 1-th time... go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da check diff successfully % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 run task successfully ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 243 100 243 0 0 6774 0 --:--:-- --:--:-- --:--:-- 6942 + synced_status='{"synced":false,"sink_checkpoint_ts":"2025-04-06 08:21:41.159","puller_resolved_ts":"2025-04-06 08:21:35.458","last_synced_ts":"2025-04-06 08:21:35.959","now_ts":"2025-04-06 08:21:43.000","info":"The data syncing is not finished, please wait"}' ++ echo '{"synced":false,"sink_checkpoint_ts":"2025-04-06' '08:21:41.159","puller_resolved_ts":"2025-04-06' '08:21:35.458","last_synced_ts":"2025-04-06' '08:21:35.959","now_ts":"2025-04-06' '08:21:43.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq .synced + status=false + '[' false '!=' false ']' ++ echo '{"synced":false,"sink_checkpoint_ts":"2025-04-06' '08:21:41.159","puller_resolved_ts":"2025-04-06' '08:21:35.458","last_synced_ts":"2025-04-06' '08:21:35.959","now_ts":"2025-04-06' '08:21:43.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq -r .info + info='The data syncing is not finished, please wait' + '[' 'The data syncing is not finished, please wait' '!=' 'The data syncing is not finished, please wait' ']' + sleep 130 go: downloading github.com/cockroachdb/errors v1.11.1 go: downloading github.com/DataDog/zstd v1.5.5 go: downloading github.com/cockroachdb/redact v1.1.5 go: downloading github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 go: downloading github.com/getsentry/sentry-go v0.27.0 go: downloading github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b go: downloading github.com/kr/pretty v0.3.1 go: downloading github.com/rogpeppe/go-internal v1.13.1 go: downloading github.com/kr/text v0.2.0 go: downloading github.com/go-logr/logr v1.4.1 go: downloading go.opentelemetry.io/otel/metric v1.24.0 go: downloading github.com/go-logr/stdr v1.2.2 Changefeed remove successfully. ID: 1ca16915-453d-4d6c-ae3b-5cef0432008b CheckpointTs: 457152631578034177 SinkURI: mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1 check_safepoint_forward http://127.0.0.1:2379 7489988635534264163 457152633937068034 457152631578034177 457152633098207233 go: downloading golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 go: downloading github.com/goccy/go-json v0.10.2 go: downloading github.com/zeebo/xxh3 v1.0.2 go: downloading github.com/google/flatbuffers v2.0.8+incompatible go: downloading github.com/andybalholm/brotli v1.0.5 go: downloading github.com/klauspost/cpuid/v2 v2.2.4 go: downloading k8s.io/apimachinery v0.29.11 go: downloading github.com/cockroachdb/errors v1.11.1 go: downloading github.com/cockroachdb/redact v1.1.5 go: downloading github.com/DataDog/zstd v1.5.5 go: downloading github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 go: downloading github.com/getsentry/sentry-go v0.27.0 go: downloading github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b go: downloading github.com/kr/pretty v0.3.1 wait process cdc.test exit for 2-th time... check diff failed 4-th time, retry later go: downloading github.com/rogpeppe/go-internal v1.13.1 go: downloading github.com/kr/text v0.2.0 go: downloading gopkg.in/inf.v0 v0.9.1 go: downloading k8s.io/klog/v2 v2.120.1 go: downloading github.com/google/gofuzz v1.2.0 go: downloading sigs.k8s.io/structured-merge-diff/v4 v4.4.1 go: downloading sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd go: downloading k8s.io/utils v0.0.0-20230726121419-3b25d923346b go: downloading golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 go: downloading github.com/goccy/go-json v0.10.2 go: downloading github.com/google/flatbuffers v2.0.8+incompatible go: downloading github.com/zeebo/xxh3 v1.0.2 go: downloading github.com/andybalholm/brotli v1.0.5 go: downloading github.com/klauspost/cpuid/v2 v2.2.4 table ddl_manager.finish_mark not exists for 40-th check, retry later wait process cdc.test exit for 3-th time... table clustered_index_test.t0 exists table clustered_index_test.t1 exists table clustered_index_test.t2 exists check table exists success check diff failed 1-th time, retry later check diff failed 2-th time, retry later run task successfully go: downloading github.com/pierrec/lz4 v2.6.1+incompatible cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Sun Apr 6 08:21:44 CST 2025] <<<<<< run test case changefeed_error success! >>>>>> Changefeed remove successfully. ID: 892434ed-01b1-4a02-b81f-d1daca93ac09 CheckpointTs: 457152634199212035 SinkURI: mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1 check_safepoint_cleared http://127.0.0.1:2379 7489988635534264163 run task successfully go: downloading github.com/pierrec/lz4 v2.6.1+incompatible wait process cdc.test exit for 1-th time... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/region_merge/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [Pipeline] } table ddl_manager.finish_mark not exists for 41-th check, retry later wait process cdc.test exit for 2-th time... check diff failed 5-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 06 Apr 2025 00:21:46 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/e684662d-022f-4a79-9ef3-0a59dba96193 {UpstreamID:7489988628380945915 Namespace:default ID:e684662d-022f-4a79-9ef3-0a59dba96193 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-06 08:21:28.032207553 +0800 CST StartTs:457152628763131907 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0041a63f0 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457152630100066306} {CheckpointTs:457152634621788167 MinTableBarrierTs:457152634621788167 AdminJobType:noop} span: {tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}, resolvedTs: 457152634621788167, checkpointTs: 457152634621788167, state: Replicating span: {tableID:112, startKey:7480000000000000ff705f720000000000fa, endKey:7480000000000000ff705f730000000000fa}, resolvedTs: 457152634621788167, checkpointTs: 457152634621788167, state: Replicating span: {tableID:116, startKey:7480000000000000ff745f720000000000fa, endKey:7480000000000000ff745f730000000000fa}, resolvedTs: 457152634621788167, checkpointTs: 457152634621788167, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/dd36f46e-f3ab-4e3f-9802-14c60ee9a46d {"id":"dd36f46e-f3ab-4e3f-9802-14c60ee9a46d","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898903} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319608790545e7 dd36f46e-f3ab-4e3f-9802-14c60ee9a46d /tidb/cdc/default/default/changefeed/info/e684662d-022f-4a79-9ef3-0a59dba96193 {"upstream-id":7489988628380945915,"namespace":"default","changefeed-id":"e684662d-022f-4a79-9ef3-0a59dba96193","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-06T08:21:28.032207553+08:00","start-ts":457152628763131907,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457152630100066306} /tidb/cdc/default/default/changefeed/status/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":457152634621788167,"min-table-barrier-ts":457152634621788167,"admin-job-type":0} /tidb/cdc/default/default/task/position/dd36f46e-f3ab-4e3f-9802-14c60ee9a46d/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489988628380945915 {"id":7489988628380945915,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/e684662d-022f-4a79-9ef3-0a59dba96193 {UpstreamID:7489988628380945915 Namespace:default ID:e684662d-022f-4a79-9ef3-0a59dba96193 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-06 08:21:28.032207553 +0800 CST StartTs:457152628763131907 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0041a63f0 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457152630100066306} {CheckpointTs:457152634621788167 MinTableBarrierTs:457152634621788167 AdminJobType:noop} span: {tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}, resolvedTs: 457152634621788167, checkpointTs: 457152634621788167, state: Replicating span: {tableID:112, startKey:7480000000000000ff705f720000000000fa, endKey:7480000000000000ff705f730000000000fa}, resolvedTs: 457152634621788167, checkpointTs: 457152634621788167, state: Replicating span: {tableID:116, startKey:7480000000000000ff745f720000000000fa, endKey:7480000000000000ff745f730000000000fa}, resolvedTs: 457152634621788167, checkpointTs: 457152634621788167, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/dd36f46e-f3ab-4e3f-9802-14c60ee9a46d {"id":"dd36f46e-f3ab-4e3f-9802-14c60ee9a46d","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898903} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319608790545e7 dd36f46e-f3ab-4e3f-9802-14c60ee9a46d /tidb/cdc/default/default/changefeed/info/e684662d-022f-4a79-9ef3-0a59dba96193 {"upstream-id":7489988628380945915,"namespace":"default","changefeed-id":"e684662d-022f-4a79-9ef3-0a59dba96193","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-06T08:21:28.032207553+08:00","start-ts":457152628763131907,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457152630100066306} /tidb/cdc/default/default/changefeed/status/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":457152634621788167,"min-table-barrier-ts":457152634621788167,"admin-job-type":0} /tidb/cdc/default/default/task/position/dd36f46e-f3ab-4e3f-9802-14c60ee9a46d/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489988628380945915 {"id":7489988628380945915,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/e684662d-022f-4a79-9ef3-0a59dba96193 {UpstreamID:7489988628380945915 Namespace:default ID:e684662d-022f-4a79-9ef3-0a59dba96193 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-06 08:21:28.032207553 +0800 CST StartTs:457152628763131907 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0041a63f0 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457152630100066306} {CheckpointTs:457152634621788167 MinTableBarrierTs:457152634621788167 AdminJobType:noop} span: {tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}, resolvedTs: 457152634621788167, checkpointTs: 457152634621788167, state: Replicating span: {tableID:112, startKey:7480000000000000ff705f720000000000fa, endKey:7480000000000000ff705f730000000000fa}, resolvedTs: 457152634621788167, checkpointTs: 457152634621788167, state: Replicating span: {tableID:116, startKey:7480000000000000ff745f720000000000fa, endKey:7480000000000000ff745f730000000000fa}, resolvedTs: 457152634621788167, checkpointTs: 457152634621788167, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/dd36f46e-f3ab-4e3f-9802-14c60ee9a46d {"id":"dd36f46e-f3ab-4e3f-9802-14c60ee9a46d","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898903} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319608790545e7 dd36f46e-f3ab-4e3f-9802-14c60ee9a46d /tidb/cdc/default/default/changefeed/info/e684662d-022f-4a79-9ef3-0a59dba96193 {"upstream-id":7489988628380945915,"namespace":"default","changefeed-id":"e684662d-022f-4a79-9ef3-0a59dba96193","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-06T08:21:28.032207553+08:00","start-ts":457152628763131907,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457152630100066306} /tidb/cdc/default/default/changefeed/status/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":457152634621788167,"min-table-barrier-ts":457152634621788167,"admin-job-type":0} /tidb/cdc/default/default/task/position/dd36f46e-f3ab-4e3f-9802-14c60ee9a46d/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489988628380945915 {"id":7489988628380945915,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x cdc.test cli capture list 2>&1 | grep '"is-owner": true' ***************** properties ***************** "insertproportion"="0" "readproportion"="0" "threadcount"="2" "mysql.host"="127.0.0.1" "updateproportion"="0" "mysql.port"="4000" "operationcount"="0" "scanproportion"="0" "readallfields"="true" "mysql.db"="testSync" "mysql.user"="root" "dotransactions"="false" "workload"="core" "requestdistribution"="uniform" "recordcount"="10" ********************************************** Run finished, takes 4.952437ms INSERT - Takes(s): 0.0, Count: 10, OPS: 3108.9, Avg(us): 862, Min(us): 498, Max(us): 2324, 95th(us): 3000, 99th(us): 3000 cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Sun Apr 6 08:21:46 CST 2025] <<<<<< run test case gc_safepoint success! >>>>>> check diff failed 3-th time, retry later go: downloading github.com/googleapis/enterprise-certificate-proxy v0.3.2 go: downloading github.com/google/s2a-go v0.1.7 go: downloading go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 go: downloading github.com/felixge/httpsnoop v1.0.4 go: downloading github.com/jmespath/go-jmespath v0.4.0 go: downloading github.com/google/s2a-go v0.1.7 go: downloading github.com/googleapis/enterprise-certificate-proxy v0.3.2 go: downloading go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 go: downloading github.com/felixge/httpsnoop v1.0.4 go: downloading github.com/jmespath/go-jmespath v0.4.0 start tidb cluster in /tmp/tidb_cdc_test/changefeed_fast_fail Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... [Pipeline] // timeout "is-owner": true, run task successfully [Pipeline] } =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/cdc_server_tips/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... Cache not saved (ws/jenkins-pingcap-tiflow-periodics_cdc_integration_mysql_test-234/tiflow-cdc already exists) check diff successfully [Pipeline] // cache =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/api_v2/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [Pipeline] } check diff failed 1-th time, retry later [Pipeline] // dir [Pipeline] } table ddl_manager.finish_mark not exists for 42-th check, retry later [Pipeline] // stage [Pipeline] } check diff failed 4-th time, retry later [Pipeline] // container [Pipeline] } start tidb cluster in /tmp/tidb_cdc_test/region_merge Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... [Pipeline] // withEnv [Pipeline] } owner pid: 3084 owner id dd36f46e-f3ab-4e3f-9802-14c60ee9a46d [Sun Apr 6 08:21:48 CST 2025] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.32003202.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_hang_up_owner.server2.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_hang_up_owner.server2 --cluster-id default --addr 127.0.0.1:8301 ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } table ddl_manager.finish_mark not exists for 43-th check, retry later check diff failed 5-th time, retry later start tidb cluster in /tmp/tidb_cdc_test/api_v2 Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... check diff successfully wait process cdc.test exit for 1-th time... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Verifying downstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/cdc_server_tips Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Sun Apr 6 08:21:51 CST 2025] <<<<<< run test case new_ci_collation success! >>>>>> Verifying downstream PD is started... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 06 Apr 2025 00:21:52 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/e684662d-022f-4a79-9ef3-0a59dba96193 {UpstreamID:7489988628380945915 Namespace:default ID:e684662d-022f-4a79-9ef3-0a59dba96193 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-06 08:21:28.032207553 +0800 CST StartTs:457152628763131907 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0043b61b0 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457152630100066306} {CheckpointTs:457152636194652173 MinTableBarrierTs:457152636194652173 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5dd605cc-7abc-4b98-a702-2b987738a677 {"id":"5dd605cc-7abc-4b98-a702-2b987738a677","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898909} /tidb/cdc/default/__cdc_meta__/capture/dd36f46e-f3ab-4e3f-9802-14c60ee9a46d {"id":"dd36f46e-f3ab-4e3f-9802-14c60ee9a46d","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898903} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319608790545e7 dd36f46e-f3ab-4e3f-9802-14c60ee9a46d /tidb/cdc/default/__cdc_meta__/owner/223196087905464a 5dd605cc-7abc-4b98-a702-2b987738a677 /tidb/cdc/default/default/changefeed/info/e684662d-022f-4a79-9ef3-0a59dba96193 {"upstream-id":7489988628380945915,"namespace":"default","changefeed-id":"e684662d-022f-4a79-9ef3-0a59dba96193","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-06T08:21:28.032207553+08:00","start-ts":457152628763131907,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457152630100066306} /tidb/cdc/default/default/changefeed/status/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":457152636194652173,"min-table-barrier-ts":457152636194652173,"admin-job-type":0} /tidb/cdc/default/default/task/position/5dd605cc-7abc-4b98-a702-2b987738a677/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/dd36f46e-f3ab-4e3f-9802-14c60ee9a46d/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489988628380945915 {"id":7489988628380945915,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/e684662d-022f-4a79-9ef3-0a59dba96193 {UpstreamID:7489988628380945915 Namespace:default ID:e684662d-022f-4a79-9ef3-0a59dba96193 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-06 08:21:28.032207553 +0800 CST StartTs:457152628763131907 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0043b61b0 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457152630100066306} {CheckpointTs:457152636194652173 MinTableBarrierTs:457152636194652173 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5dd605cc-7abc-4b98-a702-2b987738a677 {"id":"5dd605cc-7abc-4b98-a702-2b987738a677","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898909} /tidb/cdc/default/__cdc_meta__/capture/dd36f46e-f3ab-4e3f-9802-14c60ee9a46d {"id":"dd36f46e-f3ab-4e3f-9802-14c60ee9a46d","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898903} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319608790545e7 dd36f46e-f3ab-4e3f-9802-14c60ee9a46d /tidb/cdc/default/__cdc_meta__/owner/223196087905464a 5dd605cc-7abc-4b98-a702-2b987738a677 /tidb/cdc/default/default/changefeed/info/e684662d-022f-4a79-9ef3-0a59dba96193 {"upstream-id":7489988628380945915,"namespace":"default","changefeed-id":"e684662d-022f-4a79-9ef3-0a59dba96193","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-06T08:21:28.032207553+08:00","start-ts":457152628763131907,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457152630100066306} /tidb/cdc/default/default/changefeed/status/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":457152636194652173,"min-table-barrier-ts":457152636194652173,"admin-job-type":0} /tidb/cdc/default/default/task/position/5dd605cc-7abc-4b98-a702-2b987738a677/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/dd36f46e-f3ab-4e3f-9802-14c60ee9a46d/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489988628380945915 {"id":7489988628380945915,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + grep -q 'etcd info' + echo ' *** processors info ***: changefeedID: default/e684662d-022f-4a79-9ef3-0a59dba96193 {UpstreamID:7489988628380945915 Namespace:default ID:e684662d-022f-4a79-9ef3-0a59dba96193 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-06 08:21:28.032207553 +0800 CST StartTs:457152628763131907 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0043b61b0 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457152630100066306} {CheckpointTs:457152636194652173 MinTableBarrierTs:457152636194652173 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5dd605cc-7abc-4b98-a702-2b987738a677 {"id":"5dd605cc-7abc-4b98-a702-2b987738a677","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898909} /tidb/cdc/default/__cdc_meta__/capture/dd36f46e-f3ab-4e3f-9802-14c60ee9a46d {"id":"dd36f46e-f3ab-4e3f-9802-14c60ee9a46d","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898903} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319608790545e7 dd36f46e-f3ab-4e3f-9802-14c60ee9a46d /tidb/cdc/default/__cdc_meta__/owner/223196087905464a 5dd605cc-7abc-4b98-a702-2b987738a677 /tidb/cdc/default/default/changefeed/info/e684662d-022f-4a79-9ef3-0a59dba96193 {"upstream-id":7489988628380945915,"namespace":"default","changefeed-id":"e684662d-022f-4a79-9ef3-0a59dba96193","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-06T08:21:28.032207553+08:00","start-ts":457152628763131907,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457152630100066306} /tidb/cdc/default/default/changefeed/status/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":457152636194652173,"min-table-barrier-ts":457152636194652173,"admin-job-type":0} /tidb/cdc/default/default/task/position/5dd605cc-7abc-4b98-a702-2b987738a677/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/dd36f46e-f3ab-4e3f-9802-14c60ee9a46d/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489988628380945915 {"id":7489988628380945915,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list --server 'http://127.0.0.1:8301' 2>&1 | grep -v "dd36f46e-f3ab-4e3f-9802-14c60ee9a46d" | grep id table ddl_manager.finish_mark not exists for 44-th check, retry later Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit # diff Configuration. check-thread-count = 4 export-fix-sql = true check-struct-only = false [task] output-dir = "/tmp/tidb_cdc_test/consistent_replicate_ddl/sync_diff/output" source-instances = ["tidb"] target-instance = "mysql" target-check-tables = ["consistent_replicate_ddl.usertable*","consistent_replicate_ddl.check*"] [data-sources] [data-sources.tidb] host = "127.0.0.1" port = 4000 user = "root" password = "" snapshot = "457152601175097353" [data-sources.mysql] host = "127.0.0.1" port = 3306 user = "root" password = "" [2025/04/06 08:21:39.894 +08:00] [INFO] [helper.go:54] ["init log"] [file=] [level=info] [2025/04/06 08:21:39.894 +08:00] [INFO] [apply.go:78] ["set memory limit"] [memoryLimit=10737418240] [2025/04/06 08:21:39.895 +08:00] [INFO] [apply.go:94] ["Start http pprof server"] [addr=127.0.0.1:6060] [2025/04/06 08:21:39.895 +08:00] [INFO] [mem_quota.go:82] ["New memory quota"] [namespace=default] [changefeed=redo-applier] [total=1073741824] [2025/04/06 08:21:39.895 +08:00] [INFO] [redo.go:167] ["apply redo log starts"] [checkpointTs=457152599497375758] [resolvedTs=457152601175097353] [2025/04/06 08:21:39.896 +08:00] [INFO] [tz.go:35] ["Use the timezone of the TiCDC server machine"] [timezoneName=System] [timezone=Asia/Shanghai] [2025/04/06 08:21:39.896 +08:00] [WARN] [config.go:428] ["Because time-zone is not specified, the timezone of the TiCDC server will be used. We recommend that you specify the time-zone explicitly. Please make sure that the timezone of the TiCDC server, sink-uri and the downstream database are consistent. If the downstream database does not load the timezone information, you can refer to https://dev.mysql.com/doc/refman/8.0/en/mysql-tzinfo-to-sql.html."] [timezone=Asia/Shanghai] [2025/04/06 08:21:39.896 +08:00] [ERROR] [config.go:216] ["The TiDB source ID should never be set to 0. Please report it as a bug. The default value will be used: 1."] [tidbSourceID=0] [2025/04/06 08:21:39.897 +08:00] [INFO] [file.go:114] ["succeed to download and sort redo logs"] [type=ddl] [duration=2.14474ms] [2025/04/06 08:21:39.898 +08:00] [INFO] [file.go:114] ["succeed to download and sort redo logs"] [type=row] [duration=3.489469ms] [2025/04/06 08:21:39.900 +08:00] [INFO] [db_helper.go:161] ["sink uri is configured"] [dsn="normal:******@tcp(127.0.0.1:3306)/?interpolateParams=true&multiStatements=true&allow_auto_random_explicit_insert=1&charset=utf8mb4&foreign_key_checks=0&maxAllowedPacket=0&readTimeout=2m&sql_mode=%22IGNORE_SPACE%2CONLY_FULL_GROUP_BY%2CALLOW_INVALID_DATES%2CNO_ENGINE_SUBSTITUTION%2CNO_AUTO_VALUE_ON_ZERO%22&tidb_enable_external_ts_read=%22OFF%22&tidb_placement_mode=%22ignore%22&tidb_txn_mode=optimistic&time_zone=%22Asia%2FShanghai%22&timeout=2m&transaction_isolation=%22READ-COMMITTED%22&writeTimeout=2m"] [2025/04/06 08:21:39.903 +08:00] [INFO] [mysql.go:195] ["MySQL backends is created"] [changefeed=.] [workerCount=16] [forceReplicate=false] [2025/04/06 08:21:39.904 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/06 08:21:39.904 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/06 08:21:39.904 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/06 08:21:39.904 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/06 08:21:39.904 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/06 08:21:39.904 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/06 08:21:39.904 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/06 08:21:39.904 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/06 08:21:39.904 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/06 08:21:39.904 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/06 08:21:39.904 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/06 08:21:39.904 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/06 08:21:39.904 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/06 08:21:39.904 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/06 08:21:39.904 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/06 08:21:39.904 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/06 08:21:39.904 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=2] [2025/04/06 08:21:39.904 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=0] [2025/04/06 08:21:39.904 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=3] [2025/04/06 08:21:39.904 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=7] [2025/04/06 08:21:39.904 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=9] [2025/04/06 08:21:39.904 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=4] [2025/04/06 08:21:39.904 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=5] [2025/04/06 08:21:39.904 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=6] [2025/04/06 08:21:39.904 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=1] [2025/04/06 08:21:39.904 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=11] [2025/04/06 08:21:39.904 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=8] [2025/04/06 08:21:39.904 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=13] [2025/04/06 08:21:39.904 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=15] [2025/04/06 08:21:39.904 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=12] [2025/04/06 08:21:39.904 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=14] [2025/04/06 08:21:39.904 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=10] [2025/04/06 08:21:39.904 +08:00] [INFO] [tz.go:35] ["Use the timezone of the TiCDC server machine"] [timezoneName=System] [timezone=Asia/Shanghai] [2025/04/06 08:21:39.904 +08:00] [WARN] [config.go:428] ["Because time-zone is not specified, the timezone of the TiCDC server will be used. We recommend that you specify the time-zone explicitly. Please make sure that the timezone of the TiCDC server, sink-uri and the downstream database are consistent. If the downstream database does not load the timezone information, you can refer to https://dev.mysql.com/doc/refman/8.0/en/mysql-tzinfo-to-sql.html."] [timezone=Asia/Shanghai] [2025/04/06 08:21:39.904 +08:00] [ERROR] [config.go:216] ["The TiDB source ID should never be set to 0. Please report it as a bug. The default value will be used: 1."] [tidbSourceID=0] [2025/04/06 08:21:39.908 +08:00] [INFO] [db_helper.go:161] ["sink uri is configured"] [dsn="normal:******@tcp(127.0.0.1:3306)/?interpolateParams=true&multiStatements=true&allow_auto_random_explicit_insert=1&charset=utf8mb4&foreign_key_checks=0&maxAllowedPacket=0&readTimeout=2m&sql_mode=%22IGNORE_SPACE%2CONLY_FULL_GROUP_BY%2CALLOW_INVALID_DATES%2CNO_ENGINE_SUBSTITUTION%2CNO_AUTO_VALUE_ON_ZERO%22&tidb_enable_external_ts_read=%22OFF%22&tidb_placement_mode=%22ignore%22&tidb_txn_mode=optimistic&time_zone=%22Asia%2FShanghai%22&timeout=2m&transaction_isolation=%22READ-COMMITTED%22&writeTimeout=2m"] [2025/04/06 08:21:39.911 +08:00] [WARN] [mysql_ddl_sink.go:291] ["please set `has-vector-type` to be true if a column is vector type when the downstream is not TiDB or TiDB version less than specify version"] [hasVectorType=false] [supportVectorVersion=8.4.0] [2025/04/06 08:21:39.911 +08:00] [INFO] [mysql_ddl_sink.go:122] ["MySQL DDL sink is created"] [namespace=] [changefeed=] [2025/04/06 08:21:39.911 +08:00] [WARN] [redo.go:287] ["apply DDL"] [ddl="{\"StartTs\":457152599497375754,\"CommitTs\":457152599497375758,\"Query\":\"DROP TABLE `consistent_replicate_ddl`.`usertable1`\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"Schema\":\"consistent_replicate_ddl\",\"Table\":\"usertable1\",\"TableID\":116,\"IsPartition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":4,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false,\"BDRRole\":\"\",\"SQLMode\":0}"] [2025/04/06 08:21:39.913 +08:00] [ERROR] [async_ddl.go:184] ["check async exec ddl failed"] [namespace=] [changefeed=] [error="Error 8121 (HY000): privilege check for 'Super' fail"] [2025/04/06 08:21:39.913 +08:00] [INFO] [mysql_ddl_sink.go:225] ["Start exec DDL"] [namespace=] [changefeed=] [commitTs=457152599497375758] [DDL="DROP TABLE `consistent_replicate_ddl`.`usertable1`"] [2025/04/06 08:21:39.963 +08:00] [INFO] [mysql_ddl_sink.go:272] ["Exec DDL succeeded"] [namespace=] [changefeed=] [duration=49.705513ms] [sql="DROP TABLE `consistent_replicate_ddl`.`usertable1`"] [2025/04/06 08:21:39.963 +08:00] [WARN] [redo.go:287] ["apply DDL"] [ddl="{\"StartTs\":457152599628447768,\"CommitTs\":457152599628447771,\"Query\":\"RENAME TABLE consistent_replicate_ddl.usertable_bak TO consistent_replicate_ddl.usertable1\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"Schema\":\"consistent_replicate_ddl\",\"Table\":\"usertable1\",\"TableID\":122,\"IsPartition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":14,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false,\"BDRRole\":\"\",\"SQLMode\":0}"] [2025/04/06 08:21:39.963 +08:00] [ERROR] [async_ddl.go:184] ["check async exec ddl failed"] [namespace=] [changefeed=] [error="Error 8121 (HY000): privilege check for 'Super' fail"] [2025/04/06 08:21:39.964 +08:00] [INFO] [mysql_ddl_sink.go:225] ["Start exec DDL"] [namespace=] [changefeed=] [commitTs=457152599628447771] [DDL="RENAME TABLE consistent_replicate_ddl.usertable_bak TO consistent_replicate_ddl.usertable1"] [2025/04/06 08:21:39.995 +08:00] [INFO] [mysql_ddl_sink.go:272] ["Exec DDL succeeded"] [namespace=] [changefeed=] [duration=31.243498ms] [sql="RENAME TABLE consistent_replicate_ddl.usertable_bak TO consistent_replicate_ddl.usertable1"] [2025/04/06 08:21:39.995 +08:00] [WARN] [redo.go:287] ["apply DDL"] [ddl="{\"StartTs\":457152599772364823,\"CommitTs\":457152599772364827,\"Query\":\"RENAME TABLE consistent_replicate_ddl.usertable1 TO consistent_replicate_ddl.usertable1_1\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"Schema\":\"consistent_replicate_ddl\",\"Table\":\"usertable1_1\",\"TableID\":122,\"IsPartition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":14,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false,\"BDRRole\":\"\",\"SQLMode\":0}"] [2025/04/06 08:21:40.196 +08:00] [ERROR] [async_ddl.go:184] ["check async exec ddl failed"] [namespace=] [changefeed=] [error="Error 8121 (HY000): privilege check for 'Super' fail"] [2025/04/06 08:21:40.196 +08:00] [INFO] [mysql_ddl_sink.go:225] ["Start exec DDL"] [namespace=] [changefeed=] [commitTs=457152599772364827] [DDL="RENAME TABLE consistent_replicate_ddl.usertable1 TO consistent_replicate_ddl.usertable1_1"] [2025/04/06 08:21:40.227 +08:00] [INFO] [mysql_ddl_sink.go:272] ["Exec DDL succeeded"] [namespace=] [changefeed=] [duration=30.713654ms] [sql="RENAME TABLE consistent_replicate_ddl.usertable1 TO consistent_replicate_ddl.usertable1_1"] [2025/04/06 08:21:40.227 +08:00] [WARN] [redo.go:287] ["apply DDL"] [ddl="{\"StartTs\":457152599916544010,\"CommitTs\":457152599916544014,\"Query\":\"CREATE DATABASE consistent_replicate_ddl1\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"Schema\":\"consistent_replicate_ddl1\",\"Table\":\"\",\"TableID\":0,\"IsPartition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":1,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false,\"BDRRole\":\"\",\"SQLMode\":0}"] [2025/04/06 08:21:40.428 +08:00] [INFO] [mysql_ddl_sink.go:225] ["Start exec DDL"] [namespace=] [changefeed=] [commitTs=457152599916544014] [DDL="CREATE DATABASE consistent_replicate_ddl1"] [2025/04/06 08:21:40.448 +08:00] [INFO] [mysql_ddl_sink.go:272] ["Exec DDL succeeded"] [namespace=] [changefeed=] [duration=19.627201ms] [sql="CREATE DATABASE consistent_replicate_ddl1"] [2025/04/06 08:21:40.448 +08:00] [WARN] [redo.go:287] ["apply DDL"] [ddl="{\"StartTs\":457152600047878164,\"CommitTs\":457152600047878167,\"Query\":\"ALTER DATABASE consistent_replicate_ddl CHARACTER SET utf8mb4 COLLATE utf8mb4_bin\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"Schema\":\"consistent_replicate_ddl\",\"Table\":\"\",\"TableID\":0,\"IsPartition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":26,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false,\"BDRRole\":\"\",\"SQLMode\":0}"] [2025/04/06 08:21:40.448 +08:00] [ERROR] [async_ddl.go:184] ["check async exec ddl failed"] [namespace=] [changefeed=] [error="Error 8121 (HY000): privilege check for 'Super' fail"] [2025/04/06 08:21:40.448 +08:00] [INFO] [mysql_ddl_sink.go:225] ["Start exec DDL"] [namespace=] [changefeed=] [commitTs=457152600047878167] [DDL="ALTER DATABASE consistent_replicate_ddl CHARACTER SET utf8mb4 COLLATE utf8mb4_bin"] [2025/04/06 08:21:40.467 +08:00] [INFO] [mysql_ddl_sink.go:272] ["Exec DDL succeeded"] [namespace=] [changefeed=] [duration=18.593612ms] [sql="ALTER DATABASE consistent_replicate_ddl CHARACTER SET utf8mb4 COLLATE utf8mb4_bin"] [2025/04/06 08:21:40.467 +08:00] [WARN] [redo.go:287] ["apply DDL"] [ddl="{\"StartTs\":457152600191795212,\"CommitTs\":457152600191795215,\"Query\":\"ALTER TABLE consistent_replicate_ddl.usertable2 CHARACTER SET utf8mb4 COLLATE utf8mb4_bin\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"Schema\":\"consistent_replicate_ddl\",\"Table\":\"usertable2\",\"TableID\":118,\"IsPartition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":22,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false,\"BDRRole\":\"\",\"SQLMode\":0}"] [2025/04/06 08:21:40.467 +08:00] [ERROR] [async_ddl.go:184] ["check async exec ddl failed"] [namespace=] [changefeed=] [error="Error 8121 (HY000): privilege check for 'Super' fail"] [2025/04/06 08:21:40.467 +08:00] [INFO] [mysql_ddl_sink.go:225] ["Start exec DDL"] [namespace=] [changefeed=] [commitTs=457152600191795215] [DDL="ALTER TABLE consistent_replicate_ddl.usertable2 CHARACTER SET utf8mb4 COLLATE utf8mb4_bin"] [2025/04/06 08:21:40.487 +08:00] [INFO] [mysql_ddl_sink.go:272] ["Exec DDL succeeded"] [namespace=] [changefeed=] [duration=19.643903ms] [sql="ALTER TABLE consistent_replicate_ddl.usertable2 CHARACTER SET utf8mb4 COLLATE utf8mb4_bin"] [2025/04/06 08:21:40.487 +08:00] [WARN] [redo.go:287] ["apply DDL"] [ddl="{\"StartTs\":457152600349343748,\"CommitTs\":457152600349343751,\"Query\":\"ALTER TABLE consistent_replicate_ddl.usertable2 DROP COLUMN FIELD0\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"Schema\":\"consistent_replicate_ddl\",\"Table\":\"usertable2\",\"TableID\":118,\"IsPartition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":6,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false,\"BDRRole\":\"\",\"SQLMode\":0}"] [2025/04/06 08:21:40.687 +08:00] [INFO] [mysql_ddl_sink.go:225] ["Start exec DDL"] [namespace=] [changefeed=] [commitTs=457152600349343751] [DDL="ALTER TABLE consistent_replicate_ddl.usertable2 DROP COLUMN FIELD0"] [2025/04/06 08:21:40.736 +08:00] [INFO] [mysql_ddl_sink.go:272] ["Exec DDL succeeded"] [namespace=] [changefeed=] [duration=48.239831ms] [sql="ALTER TABLE consistent_replicate_ddl.usertable2 DROP COLUMN FIELD0"] [2025/04/06 08:21:40.736 +08:00] [WARN] [redo.go:287] ["apply DDL"] [ddl="{\"StartTs\":457152600480153636,\"CommitTs\":457152600480153640,\"Query\":\"ALTER TABLE consistent_replicate_ddl.usertable2 ADD COLUMN dummy varchar(30)\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"Schema\":\"consistent_replicate_ddl\",\"Table\":\"usertable2\",\"TableID\":118,\"IsPartition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":5,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false,\"BDRRole\":\"\",\"SQLMode\":0}"] [2025/04/06 08:21:40.736 +08:00] [INFO] [mysql_ddl_sink.go:225] ["Start exec DDL"] [namespace=] [changefeed=] [commitTs=457152600480153640] [DDL="ALTER TABLE consistent_replicate_ddl.usertable2 ADD COLUMN dummy varchar(30)"] [2025/04/06 08:21:40.783 +08:00] [INFO] [mysql_ddl_sink.go:272] ["Exec DDL succeeded"] [namespace=] [changefeed=] [duration=47.102035ms] [sql="ALTER TABLE consistent_replicate_ddl.usertable2 ADD COLUMN dummy varchar(30)"] [2025/04/06 08:21:40.783 +08:00] [WARN] [redo.go:287] ["apply DDL"] [ddl="{\"StartTs\":457152600611487789,\"CommitTs\":457152600624332803,\"Query\":\"DROP DATABASE consistent_replicate_ddl1\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"Schema\":\"consistent_replicate_ddl1\",\"Table\":\"\",\"TableID\":0,\"IsPartition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":2,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false,\"BDRRole\":\"\",\"SQLMode\":0}"] [2025/04/06 08:21:40.783 +08:00] [INFO] [mysql_ddl_sink.go:225] ["Start exec DDL"] [namespace=] [changefeed=] [commitTs=457152600624332803] [DDL="DROP DATABASE consistent_replicate_ddl1"] [2025/04/06 08:21:40.816 +08:00] [INFO] [mysql_ddl_sink.go:272] ["Exec DDL succeeded"] [namespace=] [changefeed=] [duration=33.34328ms] [sql="DROP DATABASE consistent_replicate_ddl1"] [2025/04/06 08:21:40.817 +08:00] [WARN] [redo.go:287] ["apply DDL"] [ddl="{\"StartTs\":457152600742559773,\"CommitTs\":457152600755404805,\"Query\":\"ALTER TABLE consistent_replicate_ddl.usertable3 CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"Schema\":\"consistent_replicate_ddl\",\"Table\":\"usertable3\",\"TableID\":120,\"IsPartition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":22,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false,\"BDRRole\":\"\",\"SQLMode\":0}"] [2025/04/06 08:21:40.817 +08:00] [ERROR] [async_ddl.go:184] ["check async exec ddl failed"] [namespace=] [changefeed=] [error="Error 8121 (HY000): privilege check for 'Super' fail"] [2025/04/06 08:21:40.817 +08:00] [INFO] [mysql_ddl_sink.go:225] ["Start exec DDL"] [namespace=] [changefeed=] [commitTs=457152600755404805] [DDL="ALTER TABLE consistent_replicate_ddl.usertable3 CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci"] [2025/04/06 08:21:40.836 +08:00] [INFO] [mysql_ddl_sink.go:272] ["Exec DDL succeeded"] [namespace=] [changefeed=] [duration=18.549655ms] [sql="ALTER TABLE consistent_replicate_ddl.usertable3 CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci"] [2025/04/06 08:21:40.836 +08:00] [WARN] [redo.go:287] ["apply DDL"] [ddl="{\"StartTs\":457152600886738975,\"CommitTs\":457152600886738979,\"Query\":\"RENAME TABLE `consistent_replicate_ddl`.`usertable3` TO `consistent_replicate_ddl`.`usertable3_1`\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"Schema\":\"consistent_replicate_ddl\",\"Table\":\"usertable3_1\",\"TableID\":120,\"IsPartition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":14,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false,\"BDRRole\":\"\",\"SQLMode\":0}"] [2025/04/06 08:21:40.836 +08:00] [ERROR] [async_ddl.go:184] ["check async exec ddl failed"] [namespace=] [changefeed=] [error="Error 8121 (HY000): privilege check for 'Super' fail"] [2025/04/06 08:21:40.836 +08:00] [INFO] [mysql_ddl_sink.go:225] ["Start exec DDL"] [namespace=] [changefeed=] [commitTs=457152600886738979] [DDL="RENAME TABLE `consistent_replicate_ddl`.`usertable3` TO `consistent_replicate_ddl`.`usertable3_1`"] [2025/04/06 08:21:40.863 +08:00] [INFO] [mysql_ddl_sink.go:272] ["Exec DDL succeeded"] [namespace=] [changefeed=] [duration=27.36222ms] [sql="RENAME TABLE `consistent_replicate_ddl`.`usertable3` TO `consistent_replicate_ddl`.`usertable3_1`"] [2025/04/06 08:21:40.863 +08:00] [WARN] [redo.go:287] ["apply DDL"] [ddl="{\"StartTs\":457152600886738975,\"CommitTs\":457152600886738979,\"Query\":\"RENAME TABLE `consistent_replicate_ddl`.`usertable2` TO `consistent_replicate_ddl`.`usertable2_1`\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"Schema\":\"consistent_replicate_ddl\",\"Table\":\"usertable2_1\",\"TableID\":118,\"IsPartition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":14,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false,\"BDRRole\":\"\",\"SQLMode\":0}"] [2025/04/06 08:21:40.864 +08:00] [ERROR] [async_ddl.go:184] ["check async exec ddl failed"] [namespace=] [changefeed=] [error="Error 8121 (HY000): privilege check for 'Super' fail"] [2025/04/06 08:21:40.864 +08:00] [INFO] [mysql_ddl_sink.go:225] ["Start exec DDL"] [namespace=] [changefeed=] [commitTs=457152600886738979] [DDL="RENAME TABLE `consistent_replicate_ddl`.`usertable2` TO `consistent_replicate_ddl`.`usertable2_1`"] [2025/04/06 08:21:40.891 +08:00] [INFO] [mysql_ddl_sink.go:272] ["Exec DDL succeeded"] [namespace=] [changefeed=] [duration=27.289012ms] [sql="RENAME TABLE `consistent_replicate_ddl`.`usertable2` TO `consistent_replicate_ddl`.`usertable2_1`"] [2025/04/06 08:21:40.891 +08:00] [WARN] [redo.go:287] ["apply DDL"] [ddl="{\"StartTs\":457152601030918146,\"CommitTs\":457152601030918150,\"Query\":\"ALTER TABLE consistent_replicate_ddl.usertable3_1 MODIFY COLUMN FIELD1 varchar(100)\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"Schema\":\"consistent_replicate_ddl\",\"Table\":\"usertable3_1\",\"TableID\":120,\"IsPartition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":12,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false,\"BDRRole\":\"\",\"SQLMode\":0}"] [2025/04/06 08:21:40.891 +08:00] [INFO] [mysql_ddl_sink.go:225] ["Start exec DDL"] [namespace=] [changefeed=] [commitTs=457152601030918150] [DDL="ALTER TABLE consistent_replicate_ddl.usertable3_1 MODIFY COLUMN FIELD1 varchar(100)"] [2025/04/06 08:21:40.910 +08:00] [INFO] [mysql_ddl_sink.go:272] ["Exec DDL succeeded"] [namespace=] [changefeed=] [duration=18.90114ms] [sql="ALTER TABLE consistent_replicate_ddl.usertable3_1 MODIFY COLUMN FIELD1 varchar(100)"] [2025/04/06 08:21:40.910 +08:00] [WARN] [redo.go:287] ["apply DDL"] [ddl="{\"StartTs\":457152601175097350,\"CommitTs\":457152601175097353,\"Query\":\"CREATE table consistent_replicate_ddl.check1(id int primary key)\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"Schema\":\"consistent_replicate_ddl\",\"Table\":\"check1\",\"TableID\":137,\"IsPartition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":3,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false,\"BDRRole\":\"\",\"SQLMode\":0}"] [2025/04/06 08:21:41.111 +08:00] [INFO] [mysql_ddl_sink.go:225] ["Start exec DDL"] [namespace=] [changefeed=] [commitTs=457152601175097353] [DDL="CREATE table consistent_replicate_ddl.check1(id int primary key)"] [2025/04/06 08:21:41.133 +08:00] [INFO] [mysql_ddl_sink.go:272] ["Exec DDL succeeded"] [namespace=] [changefeed=] [duration=21.747208ms] [sql="CREATE table consistent_replicate_ddl.check1(id int primary key)"] [2025/04/06 08:21:41.133 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [span="{tableID:122, startKey:7480000000000000ff7a5f720000000000fa, endKey:7480000000000000ff7a5f730000000000fa}"] [checkpointTs=457152601175097353] [2025/04/06 08:21:41.133 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [span="{tableID:122, startKey:7480000000000000ff7a5f720000000000fa, endKey:7480000000000000ff7a5f730000000000fa}"] [checkpointTs=457152601175097353] [2025/04/06 08:21:41.233 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [span="{tableID:118, startKey:7480000000000000ff765f720000000000fa, endKey:7480000000000000ff765f730000000000fa}"] [checkpointTs=457152601175097353] [2025/04/06 08:21:41.233 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [span="{tableID:118, startKey:7480000000000000ff765f720000000000fa, endKey:7480000000000000ff765f730000000000fa}"] [checkpointTs=457152601175097353] [2025/04/06 08:21:41.333 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [span="{tableID:120, startKey:7480000000000000ff785f720000000000fa, endKey:7480000000000000ff785f730000000000fa}"] [checkpointTs=457152601175097353] [2025/04/06 08:21:41.333 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [span="{tableID:120, startKey:7480000000000000ff785f720000000000fa, endKey:7480000000000000ff785f730000000000fa}"] [checkpointTs=457152601175097353] [2025/04/06 08:21:41.433 +08:00] [INFO] [redo.go:223] ["apply redo log finishes"] [appliedLogCount=101] [appliedDDLCount=14] [currentCheckpoint=457152601175097353] [2025/04/06 08:21:41.433 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=0] [2025/04/06 08:21:41.433 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=15] [2025/04/06 08:21:41.433 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=10] [2025/04/06 08:21:41.433 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=6] [2025/04/06 08:21:41.433 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=13] [2025/04/06 08:21:41.433 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=14] [2025/04/06 08:21:41.433 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=5] [2025/04/06 08:21:41.433 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=8] [2025/04/06 08:21:41.433 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=3] [2025/04/06 08:21:41.433 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=2] [2025/04/06 08:21:41.433 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=9] [2025/04/06 08:21:41.433 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=7] [2025/04/06 08:21:41.433 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=1] [2025/04/06 08:21:41.433 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=12] [2025/04/06 08:21:41.433 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=4] [2025/04/06 08:21:41.433 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=11] Apply redo log successfully table consistent_replicate_ddl.check1 exists check diff successfully [Sun Apr 6 08:21:41 CST 2025] <<<<<< run test case consistent_replicate_ddl success! >>>>>> count(*) 50 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/consistent_replicate_gbk/run.sh using Sink-Type: mysql... <<================= * About to connect() to 127.0.0.1 port 24927 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:24927; Connection refused * Closing connection 0 You are running an older version of MinIO released 4 years ago Update: Run `mc admin update` Attempting encryption of all config, IAM users and policies on MinIO backend Endpoint: http://127.0.0.1:24927 Object API (Amazon S3 compatible): Go: https://docs.min.io/docs/golang-client-quickstart-guide Java: https://docs.min.io/docs/java-client-quickstart-guide Python: https://docs.min.io/docs/python-client-quickstart-guide JavaScript: https://docs.min.io/docs/javascript-client-quickstart-guide .NET: https://docs.min.io/docs/dotnet-client-quickstart-guide check diff successfully check diff failed 6-th time, retry later wait process cdc.test exit for 1-th time... * About to connect() to 127.0.0.1 port 24927 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 24927 (#0) > GET / HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:24927 > Accept: */* > < HTTP/1.1 403 Forbidden < Accept-Ranges: bytes < Content-Length: 226 < Content-Security-Policy: block-all-mixed-content < Content-Type: application/xml < Server: MinIO/RELEASE.2020-07-27T18-37-02Z < Vary: Origin < X-Amz-Request-Id: 183392D274ADDCCB < X-Xss-Protection: 1; mode=block < Date: Sun, 06 Apr 2025 00:21:53 GMT < { [data not shown] * Connection #0 to host 127.0.0.1 left intact Bucket 's3://logbucket/' created The 1 times to try to start tidb cluster... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release "cluster-id": "default" "id": "5dd605cc-7abc-4b98-a702-2b987738a677", "cluster-id": "default" run task successfully wait process cdc.test exit for 2-th time... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table ddl_manager.finish_mark not exists for 45-th check, retry later Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Sun Apr 6 08:21:54 CST 2025] <<<<<< run test case owner_resign success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) capture_id: 5dd605cc-7abc-4b98-a702-2b987738a677 ETCDCTL_API=3 etcdctl get /tidb/cdc/default/__cdc_meta__/owner --prefix | grep '5dd605cc-7abc-4b98-a702-2b987738a677' 5dd605cc-7abc-4b98-a702-2b987738a677 run task successfully test_hang_up_owner: pass check diff failed 7-th time, retry later [2025/04/06 08:21:53.411 +08:00] [INFO] [case.go:450] ["open db success"] [dsn=root@tcp(127.0.0.1:4000)/bank] [2025/04/06 08:21:53.413 +08:00] [INFO] [case.go:450] ["open db success"] [dsn=root@tcp(127.0.0.1:3306)/bank] [2025/04/06 08:21:53.413 +08:00] [INFO] [case.go:388] ["drop tables"] [table=accounts_seq0] [2025/04/06 08:21:53.931 +08:00] [INFO] [case.go:370] ["insert 100 takes 7.181958ms"] [query="INSERT IGNORE INTO accounts_seq0 (id, counter, sequence, startts) VALUES (0, 0, 0, 0),(1, 0, 0, 0),(2, 0, 0, 0),(3, 0, 0, 0),(4, 0, 0, 0),(5, 0, 0, 0),(6, 0, 0, 0),(7, 0, 0, 0),(8, 0, 0, 0),(9, 0, 0, 0),(10, 0, 0, 0),(11, 0, 0, 0),(12, 0, 0, 0),(13, 0, 0, 0),(14, 0, 0, 0),(15, 0, 0, 0),(16, 0, 0, 0),(17, 0, 0, 0),(18, 0, 0, 0),(19, 0, 0, 0),(20, 0, 0, 0),(21, 0, 0, 0),(22, 0, 0, 0),(23, 0, 0, 0),(24, 0, 0, 0),(25, 0, 0, 0),(26, 0, 0, 0),(27, 0, 0, 0),(28, 0, 0, 0),(29, 0, 0, 0),(30, 0, 0, 0),(31, 0, 0, 0),(32, 0, 0, 0),(33, 0, 0, 0),(34, 0, 0, 0),(35, 0, 0, 0),(36, 0, 0, 0),(37, 0, 0, 0),(38, 0, 0, 0),(39, 0, 0, 0),(40, 0, 0, 0),(41, 0, 0, 0),(42, 0, 0, 0),(43, 0, 0, 0),(44, 0, 0, 0),(45, 0, 0, 0),(46, 0, 0, 0),(47, 0, 0, 0),(48, 0, 0, 0),(49, 0, 0, 0),(50, 0, 0, 0),(51, 0, 0, 0),(52, 0, 0, 0),(53, 0, 0, 0),(54, 0, 0, 0),(55, 0, 0, 0),(56, 0, 0, 0),(57, 0, 0, 0),(58, 0, 0, 0),(59, 0, 0, 0),(60, 0, 0, 0),(61, 0, 0, 0),(62, 0, 0, 0),(63, 0, 0, 0),(64, 0, 0, 0),(65, 0, 0, 0),(66, 0, 0, 0),(67, 0, 0, 0),(68, 0, 0, 0),(69, 0, 0, 0),(70, 0, 0, 0),(71, 0, 0, 0),(72, 0, 0, 0),(73, 0, 0, 0),(74, 0, 0, 0),(75, 0, 0, 0),(76, 0, 0, 0),(77, 0, 0, 0),(78, 0, 0, 0),(79, 0, 0, 0),(80, 0, 0, 0),(81, 0, 0, 0),(82, 0, 0, 0),(83, 0, 0, 0),(84, 0, 0, 0),(85, 0, 0, 0),(86, 0, 0, 0),(87, 0, 0, 0),(88, 0, 0, 0),(89, 0, 0, 0),(90, 0, 0, 0),(91, 0, 0, 0),(92, 0, 0, 0),(93, 0, 0, 0),(94, 0, 0, 0),(95, 0, 0, 0),(96, 0, 0, 0),(97, 0, 0, 0),(98, 0, 0, 0),(99, 0, 0, 0)"] [2025/04/06 08:21:53.932 +08:00] [INFO] [case.go:388] ["drop tables"] [table=accounts_seq1] [2025/04/06 08:21:54.451 +08:00] [INFO] [case.go:370] ["insert 100 takes 8.736296ms"] [query="INSERT IGNORE INTO accounts_seq1 (id, counter, sequence, startts) VALUES (0, 0, 0, 0),(1, 0, 0, 0),(2, 0, 0, 0),(3, 0, 0, 0),(4, 0, 0, 0),(5, 0, 0, 0),(6, 0, 0, 0),(7, 0, 0, 0),(8, 0, 0, 0),(9, 0, 0, 0),(10, 0, 0, 0),(11, 0, 0, 0),(12, 0, 0, 0),(13, 0, 0, 0),(14, 0, 0, 0),(15, 0, 0, 0),(16, 0, 0, 0),(17, 0, 0, 0),(18, 0, 0, 0),(19, 0, 0, 0),(20, 0, 0, 0),(21, 0, 0, 0),(22, 0, 0, 0),(23, 0, 0, 0),(24, 0, 0, 0),(25, 0, 0, 0),(26, 0, 0, 0),(27, 0, 0, 0),(28, 0, 0, 0),(29, 0, 0, 0),(30, 0, 0, 0),(31, 0, 0, 0),(32, 0, 0, 0),(33, 0, 0, 0),(34, 0, 0, 0),(35, 0, 0, 0),(36, 0, 0, 0),(37, 0, 0, 0),(38, 0, 0, 0),(39, 0, 0, 0),(40, 0, 0, 0),(41, 0, 0, 0),(42, 0, 0, 0),(43, 0, 0, 0),(44, 0, 0, 0),(45, 0, 0, 0),(46, 0, 0, 0),(47, 0, 0, 0),(48, 0, 0, 0),(49, 0, 0, 0),(50, 0, 0, 0),(51, 0, 0, 0),(52, 0, 0, 0),(53, 0, 0, 0),(54, 0, 0, 0),(55, 0, 0, 0),(56, 0, 0, 0),(57, 0, 0, 0),(58, 0, 0, 0),(59, 0, 0, 0),(60, 0, 0, 0),(61, 0, 0, 0),(62, 0, 0, 0),(63, 0, 0, 0),(64, 0, 0, 0),(65, 0, 0, 0),(66, 0, 0, 0),(67, 0, 0, 0),(68, 0, 0, 0),(69, 0, 0, 0),(70, 0, 0, 0),(71, 0, 0, 0),(72, 0, 0, 0),(73, 0, 0, 0),(74, 0, 0, 0),(75, 0, 0, 0),(76, 0, 0, 0),(77, 0, 0, 0),(78, 0, 0, 0),(79, 0, 0, 0),(80, 0, 0, 0),(81, 0, 0, 0),(82, 0, 0, 0),(83, 0, 0, 0),(84, 0, 0, 0),(85, 0, 0, 0),(86, 0, 0, 0),(87, 0, 0, 0),(88, 0, 0, 0),(89, 0, 0, 0),(90, 0, 0, 0),(91, 0, 0, 0),(92, 0, 0, 0),(93, 0, 0, 0),(94, 0, 0, 0),(95, 0, 0, 0),(96, 0, 0, 0),(97, 0, 0, 0),(98, 0, 0, 0),(99, 0, 0, 0)"] [2025/04/06 08:21:54.451 +08:00] [INFO] [case.go:388] ["drop tables"] [table=accounts_seq2] [2025/04/06 08:21:54.968 +08:00] [INFO] [case.go:370] ["insert 100 takes 6.302193ms"] [query="INSERT IGNORE INTO accounts_seq2 (id, counter, sequence, startts) VALUES (0, 0, 0, 0),(1, 0, 0, 0),(2, 0, 0, 0),(3, 0, 0, 0),(4, 0, 0, 0),(5, 0, 0, 0),(6, 0, 0, 0),(7, 0, 0, 0),(8, 0, 0, 0),(9, 0, 0, 0),(10, 0, 0, 0),(11, 0, 0, 0),(12, 0, 0, 0),(13, 0, 0, 0),(14, 0, 0, 0),(15, 0, 0, 0),(16, 0, 0, 0),(17, 0, 0, 0),(18, 0, 0, 0),(19, 0, 0, 0),(20, 0, 0, 0),(21, 0, 0, 0),(22, 0, 0, 0),(23, 0, 0, 0),(24, 0, 0, 0),(25, 0, 0, 0),(26, 0, 0, 0),(27, 0, 0, 0),(28, 0, 0, 0),(29, 0, 0, 0),(30, 0, 0, 0),(31, 0, 0, 0),(32, 0, 0, 0),(33, 0, 0, 0),(34, 0, 0, 0),(35, 0, 0, 0),(36, 0, 0, 0),(37, 0, 0, 0),(38, 0, 0, 0),(39, 0, 0, 0),(40, 0, 0, 0),(41, 0, 0, 0),(42, 0, 0, 0),(43, 0, 0, 0),(44, 0, 0, 0),(45, 0, 0, 0),(46, 0, 0, 0),(47, 0, 0, 0),(48, 0, 0, 0),(49, 0, 0, 0),(50, 0, 0, 0),(51, 0, 0, 0),(52, 0, 0, 0),(53, 0, 0, 0),(54, 0, 0, 0),(55, 0, 0, 0),(56, 0, 0, 0),(57, 0, 0, 0),(58, 0, 0, 0),(59, 0, 0, 0),(60, 0, 0, 0),(61, 0, 0, 0),(62, 0, 0, 0),(63, 0, 0, 0),(64, 0, 0, 0),(65, 0, 0, 0),(66, 0, 0, 0),(67, 0, 0, 0),(68, 0, 0, 0),(69, 0, 0, 0),(70, 0, 0, 0),(71, 0, 0, 0),(72, 0, 0, 0),(73, 0, 0, 0),(74, 0, 0, 0),(75, 0, 0, 0),(76, 0, 0, 0),(77, 0, 0, 0),(78, 0, 0, 0),(79, 0, 0, 0),(80, 0, 0, 0),(81, 0, 0, 0),(82, 0, 0, 0),(83, 0, 0, 0),(84, 0, 0, 0),(85, 0, 0, 0),(86, 0, 0, 0),(87, 0, 0, 0),(88, 0, 0, 0),(89, 0, 0, 0),(90, 0, 0, 0),(91, 0, 0, 0),(92, 0, 0, 0),(93, 0, 0, 0),(94, 0, 0, 0),(95, 0, 0, 0),(96, 0, 0, 0),(97, 0, 0, 0),(98, 0, 0, 0),(99, 0, 0, 0)"] [2025/04/06 08:21:54.968 +08:00] [INFO] [case.go:388] ["drop tables"] [table=accounts_seq3] Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 1-th time... [2025/04/06 08:21:55.484 +08:00] [INFO] [case.go:370] ["insert 100 takes 6.344621ms"] [query="INSERT IGNORE INTO accounts_seq3 (id, counter, sequence, startts) VALUES (0, 0, 0, 0),(1, 0, 0, 0),(2, 0, 0, 0),(3, 0, 0, 0),(4, 0, 0, 0),(5, 0, 0, 0),(6, 0, 0, 0),(7, 0, 0, 0),(8, 0, 0, 0),(9, 0, 0, 0),(10, 0, 0, 0),(11, 0, 0, 0),(12, 0, 0, 0),(13, 0, 0, 0),(14, 0, 0, 0),(15, 0, 0, 0),(16, 0, 0, 0),(17, 0, 0, 0),(18, 0, 0, 0),(19, 0, 0, 0),(20, 0, 0, 0),(21, 0, 0, 0),(22, 0, 0, 0),(23, 0, 0, 0),(24, 0, 0, 0),(25, 0, 0, 0),(26, 0, 0, 0),(27, 0, 0, 0),(28, 0, 0, 0),(29, 0, 0, 0),(30, 0, 0, 0),(31, 0, 0, 0),(32, 0, 0, 0),(33, 0, 0, 0),(34, 0, 0, 0),(35, 0, 0, 0),(36, 0, 0, 0),(37, 0, 0, 0),(38, 0, 0, 0),(39, 0, 0, 0),(40, 0, 0, 0),(41, 0, 0, 0),(42, 0, 0, 0),(43, 0, 0, 0),(44, 0, 0, 0),(45, 0, 0, 0),(46, 0, 0, 0),(47, 0, 0, 0),(48, 0, 0, 0),(49, 0, 0, 0),(50, 0, 0, 0),(51, 0, 0, 0),(52, 0, 0, 0),(53, 0, 0, 0),(54, 0, 0, 0),(55, 0, 0, 0),(56, 0, 0, 0),(57, 0, 0, 0),(58, 0, 0, 0),(59, 0, 0, 0),(60, 0, 0, 0),(61, 0, 0, 0),(62, 0, 0, 0),(63, 0, 0, 0),(64, 0, 0, 0),(65, 0, 0, 0),(66, 0, 0, 0),(67, 0, 0, 0),(68, 0, 0, 0),(69, 0, 0, 0),(70, 0, 0, 0),(71, 0, 0, 0),(72, 0, 0, 0),(73, 0, 0, 0),(74, 0, 0, 0),(75, 0, 0, 0),(76, 0, 0, 0),(77, 0, 0, 0),(78, 0, 0, 0),(79, 0, 0, 0),(80, 0, 0, 0),(81, 0, 0, 0),(82, 0, 0, 0),(83, 0, 0, 0),(84, 0, 0, 0),(85, 0, 0, 0),(86, 0, 0, 0),(87, 0, 0, 0),(88, 0, 0, 0),(89, 0, 0, 0),(90, 0, 0, 0),(91, 0, 0, 0),(92, 0, 0, 0),(93, 0, 0, 0),(94, 0, 0, 0),(95, 0, 0, 0),(96, 0, 0, 0),(97, 0, 0, 0),(98, 0, 0, 0),(99, 0, 0, 0)"] [2025/04/06 08:21:55.484 +08:00] [INFO] [case.go:388] ["drop tables"] [table=accounts_seq4] Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 46-th check, retry later wait process cdc.test exit for 2-th time... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2025/04/06 08:21:56.001 +08:00] [INFO] [case.go:370] ["insert 100 takes 5.933292ms"] [query="INSERT IGNORE INTO accounts_seq4 (id, counter, sequence, startts) VALUES (0, 0, 0, 0),(1, 0, 0, 0),(2, 0, 0, 0),(3, 0, 0, 0),(4, 0, 0, 0),(5, 0, 0, 0),(6, 0, 0, 0),(7, 0, 0, 0),(8, 0, 0, 0),(9, 0, 0, 0),(10, 0, 0, 0),(11, 0, 0, 0),(12, 0, 0, 0),(13, 0, 0, 0),(14, 0, 0, 0),(15, 0, 0, 0),(16, 0, 0, 0),(17, 0, 0, 0),(18, 0, 0, 0),(19, 0, 0, 0),(20, 0, 0, 0),(21, 0, 0, 0),(22, 0, 0, 0),(23, 0, 0, 0),(24, 0, 0, 0),(25, 0, 0, 0),(26, 0, 0, 0),(27, 0, 0, 0),(28, 0, 0, 0),(29, 0, 0, 0),(30, 0, 0, 0),(31, 0, 0, 0),(32, 0, 0, 0),(33, 0, 0, 0),(34, 0, 0, 0),(35, 0, 0, 0),(36, 0, 0, 0),(37, 0, 0, 0),(38, 0, 0, 0),(39, 0, 0, 0),(40, 0, 0, 0),(41, 0, 0, 0),(42, 0, 0, 0),(43, 0, 0, 0),(44, 0, 0, 0),(45, 0, 0, 0),(46, 0, 0, 0),(47, 0, 0, 0),(48, 0, 0, 0),(49, 0, 0, 0),(50, 0, 0, 0),(51, 0, 0, 0),(52, 0, 0, 0),(53, 0, 0, 0),(54, 0, 0, 0),(55, 0, 0, 0),(56, 0, 0, 0),(57, 0, 0, 0),(58, 0, 0, 0),(59, 0, 0, 0),(60, 0, 0, 0),(61, 0, 0, 0),(62, 0, 0, 0),(63, 0, 0, 0),(64, 0, 0, 0),(65, 0, 0, 0),(66, 0, 0, 0),(67, 0, 0, 0),(68, 0, 0, 0),(69, 0, 0, 0),(70, 0, 0, 0),(71, 0, 0, 0),(72, 0, 0, 0),(73, 0, 0, 0),(74, 0, 0, 0),(75, 0, 0, 0),(76, 0, 0, 0),(77, 0, 0, 0),(78, 0, 0, 0),(79, 0, 0, 0),(80, 0, 0, 0),(81, 0, 0, 0),(82, 0, 0, 0),(83, 0, 0, 0),(84, 0, 0, 0),(85, 0, 0, 0),(86, 0, 0, 0),(87, 0, 0, 0),(88, 0, 0, 0),(89, 0, 0, 0),(90, 0, 0, 0),(91, 0, 0, 0),(92, 0, 0, 0),(93, 0, 0, 0),(94, 0, 0, 0),(95, 0, 0, 0),(96, 0, 0, 0),(97, 0, 0, 0),(98, 0, 0, 0),(99, 0, 0, 0)"] [2025/04/06 08:21:56.001 +08:00] [INFO] [case.go:388] ["drop tables"] [table=accounts0] cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit run test case test_expire_owner [Sun Apr 6 08:21:56 CST 2025] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.33523354.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_expire_owner.server1.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_expire_owner.server1 --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2025/04/06 08:21:56.518 +08:00] [INFO] [case.go:370] ["insert 100 takes 5.700378ms"] [query="INSERT IGNORE INTO accounts0 (id, balance, startts) VALUES (0, 1000, 0),(1, 1000, 0),(2, 1000, 0),(3, 1000, 0),(4, 1000, 0),(5, 1000, 0),(6, 1000, 0),(7, 1000, 0),(8, 1000, 0),(9, 1000, 0),(10, 1000, 0),(11, 1000, 0),(12, 1000, 0),(13, 1000, 0),(14, 1000, 0),(15, 1000, 0),(16, 1000, 0),(17, 1000, 0),(18, 1000, 0),(19, 1000, 0),(20, 1000, 0),(21, 1000, 0),(22, 1000, 0),(23, 1000, 0),(24, 1000, 0),(25, 1000, 0),(26, 1000, 0),(27, 1000, 0),(28, 1000, 0),(29, 1000, 0),(30, 1000, 0),(31, 1000, 0),(32, 1000, 0),(33, 1000, 0),(34, 1000, 0),(35, 1000, 0),(36, 1000, 0),(37, 1000, 0),(38, 1000, 0),(39, 1000, 0),(40, 1000, 0),(41, 1000, 0),(42, 1000, 0),(43, 1000, 0),(44, 1000, 0),(45, 1000, 0),(46, 1000, 0),(47, 1000, 0),(48, 1000, 0),(49, 1000, 0),(50, 1000, 0),(51, 1000, 0),(52, 1000, 0),(53, 1000, 0),(54, 1000, 0),(55, 1000, 0),(56, 1000, 0),(57, 1000, 0),(58, 1000, 0),(59, 1000, 0),(60, 1000, 0),(61, 1000, 0),(62, 1000, 0),(63, 1000, 0),(64, 1000, 0),(65, 1000, 0),(66, 1000, 0),(67, 1000, 0),(68, 1000, 0),(69, 1000, 0),(70, 1000, 0),(71, 1000, 0),(72, 1000, 0),(73, 1000, 0),(74, 1000, 0),(75, 1000, 0),(76, 1000, 0),(77, 1000, 0),(78, 1000, 0),(79, 1000, 0),(80, 1000, 0),(81, 1000, 0),(82, 1000, 0),(83, 1000, 0),(84, 1000, 0),(85, 1000, 0),(86, 1000, 0),(87, 1000, 0),(88, 1000, 0),(89, 1000, 0),(90, 1000, 0),(91, 1000, 0),(92, 1000, 0),(93, 1000, 0),(94, 1000, 0),(95, 1000, 0),(96, 1000, 0),(97, 1000, 0),(98, 1000, 0),(99, 1000, 0)"] [2025/04/06 08:21:56.519 +08:00] [INFO] [case.go:388] ["drop tables"] [table=accounts1] check diff failed 8-th time, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/ddl_sequence/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489988783608874346 TiDB Cluster ID. tikv_gc_leader_uuid 65821e6c200000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-4-x87nv, pid:1215, start at 2025-04-06 08:21:54.985177992 +0800 CST m=+2.543168175 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:23:55.031 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:21:54.995 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:11:54.995 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489988783608874346 TiDB Cluster ID. tikv_gc_leader_uuid 65821e6c200000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-4-x87nv, pid:1215, start at 2025-04-06 08:21:54.985177992 +0800 CST m=+2.543168175 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:23:55.031 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:21:54.995 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:11:54.995 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489988782798603682 TiDB Cluster ID. tikv_gc_leader_uuid 65821e6c4e80003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-4-x87nv, pid:1282, start at 2025-04-06 08:21:55.133929147 +0800 CST m=+2.602174577 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:23:55.145 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:21:55.130 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:11:55.130 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/changefeed_fast_fail/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/changefeed_fast_fail/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/changefeed_fast_fail/tiflash-proxy.toml"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/changefeed_fast_fail/tiflash/db/proxy"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/changefeed_fast_fail/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [2025/04/06 08:21:57.036 +08:00] [INFO] [case.go:370] ["insert 100 takes 5.366574ms"] [query="INSERT IGNORE INTO accounts1 (id, balance, startts) VALUES (0, 1000, 0),(1, 1000, 0),(2, 1000, 0),(3, 1000, 0),(4, 1000, 0),(5, 1000, 0),(6, 1000, 0),(7, 1000, 0),(8, 1000, 0),(9, 1000, 0),(10, 1000, 0),(11, 1000, 0),(12, 1000, 0),(13, 1000, 0),(14, 1000, 0),(15, 1000, 0),(16, 1000, 0),(17, 1000, 0),(18, 1000, 0),(19, 1000, 0),(20, 1000, 0),(21, 1000, 0),(22, 1000, 0),(23, 1000, 0),(24, 1000, 0),(25, 1000, 0),(26, 1000, 0),(27, 1000, 0),(28, 1000, 0),(29, 1000, 0),(30, 1000, 0),(31, 1000, 0),(32, 1000, 0),(33, 1000, 0),(34, 1000, 0),(35, 1000, 0),(36, 1000, 0),(37, 1000, 0),(38, 1000, 0),(39, 1000, 0),(40, 1000, 0),(41, 1000, 0),(42, 1000, 0),(43, 1000, 0),(44, 1000, 0),(45, 1000, 0),(46, 1000, 0),(47, 1000, 0),(48, 1000, 0),(49, 1000, 0),(50, 1000, 0),(51, 1000, 0),(52, 1000, 0),(53, 1000, 0),(54, 1000, 0),(55, 1000, 0),(56, 1000, 0),(57, 1000, 0),(58, 1000, 0),(59, 1000, 0),(60, 1000, 0),(61, 1000, 0),(62, 1000, 0),(63, 1000, 0),(64, 1000, 0),(65, 1000, 0),(66, 1000, 0),(67, 1000, 0),(68, 1000, 0),(69, 1000, 0),(70, 1000, 0),(71, 1000, 0),(72, 1000, 0),(73, 1000, 0),(74, 1000, 0),(75, 1000, 0),(76, 1000, 0),(77, 1000, 0),(78, 1000, 0),(79, 1000, 0),(80, 1000, 0),(81, 1000, 0),(82, 1000, 0),(83, 1000, 0),(84, 1000, 0),(85, 1000, 0),(86, 1000, 0),(87, 1000, 0),(88, 1000, 0),(89, 1000, 0),(90, 1000, 0),(91, 1000, 0),(92, 1000, 0),(93, 1000, 0),(94, 1000, 0),(95, 1000, 0),(96, 1000, 0),(97, 1000, 0),(98, 1000, 0),(99, 1000, 0)"] [2025/04/06 08:21:57.036 +08:00] [INFO] [case.go:388] ["drop tables"] [table=accounts2] start tidb cluster in /tmp/tidb_cdc_test/consistent_replicate_gbk Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... [2025/04/06 08:21:57.551 +08:00] [INFO] [case.go:370] ["insert 100 takes 5.87109ms"] [query="INSERT IGNORE INTO accounts2 (id, balance, startts) VALUES (0, 1000, 0),(1, 1000, 0),(2, 1000, 0),(3, 1000, 0),(4, 1000, 0),(5, 1000, 0),(6, 1000, 0),(7, 1000, 0),(8, 1000, 0),(9, 1000, 0),(10, 1000, 0),(11, 1000, 0),(12, 1000, 0),(13, 1000, 0),(14, 1000, 0),(15, 1000, 0),(16, 1000, 0),(17, 1000, 0),(18, 1000, 0),(19, 1000, 0),(20, 1000, 0),(21, 1000, 0),(22, 1000, 0),(23, 1000, 0),(24, 1000, 0),(25, 1000, 0),(26, 1000, 0),(27, 1000, 0),(28, 1000, 0),(29, 1000, 0),(30, 1000, 0),(31, 1000, 0),(32, 1000, 0),(33, 1000, 0),(34, 1000, 0),(35, 1000, 0),(36, 1000, 0),(37, 1000, 0),(38, 1000, 0),(39, 1000, 0),(40, 1000, 0),(41, 1000, 0),(42, 1000, 0),(43, 1000, 0),(44, 1000, 0),(45, 1000, 0),(46, 1000, 0),(47, 1000, 0),(48, 1000, 0),(49, 1000, 0),(50, 1000, 0),(51, 1000, 0),(52, 1000, 0),(53, 1000, 0),(54, 1000, 0),(55, 1000, 0),(56, 1000, 0),(57, 1000, 0),(58, 1000, 0),(59, 1000, 0),(60, 1000, 0),(61, 1000, 0),(62, 1000, 0),(63, 1000, 0),(64, 1000, 0),(65, 1000, 0),(66, 1000, 0),(67, 1000, 0),(68, 1000, 0),(69, 1000, 0),(70, 1000, 0),(71, 1000, 0),(72, 1000, 0),(73, 1000, 0),(74, 1000, 0),(75, 1000, 0),(76, 1000, 0),(77, 1000, 0),(78, 1000, 0),(79, 1000, 0),(80, 1000, 0),(81, 1000, 0),(82, 1000, 0),(83, 1000, 0),(84, 1000, 0),(85, 1000, 0),(86, 1000, 0),(87, 1000, 0),(88, 1000, 0),(89, 1000, 0),(90, 1000, 0),(91, 1000, 0),(92, 1000, 0),(93, 1000, 0),(94, 1000, 0),(95, 1000, 0),(96, 1000, 0),(97, 1000, 0),(98, 1000, 0),(99, 1000, 0)"] [2025/04/06 08:21:57.552 +08:00] [INFO] [case.go:388] ["drop tables"] [table=accounts3] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2025/04/06 08:21:58.066 +08:00] [INFO] [case.go:370] ["insert 100 takes 6.134137ms"] [query="INSERT IGNORE INTO accounts3 (id, balance, startts) VALUES (0, 1000, 0),(1, 1000, 0),(2, 1000, 0),(3, 1000, 0),(4, 1000, 0),(5, 1000, 0),(6, 1000, 0),(7, 1000, 0),(8, 1000, 0),(9, 1000, 0),(10, 1000, 0),(11, 1000, 0),(12, 1000, 0),(13, 1000, 0),(14, 1000, 0),(15, 1000, 0),(16, 1000, 0),(17, 1000, 0),(18, 1000, 0),(19, 1000, 0),(20, 1000, 0),(21, 1000, 0),(22, 1000, 0),(23, 1000, 0),(24, 1000, 0),(25, 1000, 0),(26, 1000, 0),(27, 1000, 0),(28, 1000, 0),(29, 1000, 0),(30, 1000, 0),(31, 1000, 0),(32, 1000, 0),(33, 1000, 0),(34, 1000, 0),(35, 1000, 0),(36, 1000, 0),(37, 1000, 0),(38, 1000, 0),(39, 1000, 0),(40, 1000, 0),(41, 1000, 0),(42, 1000, 0),(43, 1000, 0),(44, 1000, 0),(45, 1000, 0),(46, 1000, 0),(47, 1000, 0),(48, 1000, 0),(49, 1000, 0),(50, 1000, 0),(51, 1000, 0),(52, 1000, 0),(53, 1000, 0),(54, 1000, 0),(55, 1000, 0),(56, 1000, 0),(57, 1000, 0),(58, 1000, 0),(59, 1000, 0),(60, 1000, 0),(61, 1000, 0),(62, 1000, 0),(63, 1000, 0),(64, 1000, 0),(65, 1000, 0),(66, 1000, 0),(67, 1000, 0),(68, 1000, 0),(69, 1000, 0),(70, 1000, 0),(71, 1000, 0),(72, 1000, 0),(73, 1000, 0),(74, 1000, 0),(75, 1000, 0),(76, 1000, 0),(77, 1000, 0),(78, 1000, 0),(79, 1000, 0),(80, 1000, 0),(81, 1000, 0),(82, 1000, 0),(83, 1000, 0),(84, 1000, 0),(85, 1000, 0),(86, 1000, 0),(87, 1000, 0),(88, 1000, 0),(89, 1000, 0),(90, 1000, 0),(91, 1000, 0),(92, 1000, 0),(93, 1000, 0),(94, 1000, 0),(95, 1000, 0),(96, 1000, 0),(97, 1000, 0),(98, 1000, 0),(99, 1000, 0)"] [2025/04/06 08:21:58.067 +08:00] [INFO] [case.go:388] ["drop tables"] [table=accounts4] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 47-th check, retry later [2025/04/06 08:21:58.582 +08:00] [INFO] [case.go:370] ["insert 100 takes 5.452142ms"] [query="INSERT IGNORE INTO accounts4 (id, balance, startts) VALUES (0, 1000, 0),(1, 1000, 0),(2, 1000, 0),(3, 1000, 0),(4, 1000, 0),(5, 1000, 0),(6, 1000, 0),(7, 1000, 0),(8, 1000, 0),(9, 1000, 0),(10, 1000, 0),(11, 1000, 0),(12, 1000, 0),(13, 1000, 0),(14, 1000, 0),(15, 1000, 0),(16, 1000, 0),(17, 1000, 0),(18, 1000, 0),(19, 1000, 0),(20, 1000, 0),(21, 1000, 0),(22, 1000, 0),(23, 1000, 0),(24, 1000, 0),(25, 1000, 0),(26, 1000, 0),(27, 1000, 0),(28, 1000, 0),(29, 1000, 0),(30, 1000, 0),(31, 1000, 0),(32, 1000, 0),(33, 1000, 0),(34, 1000, 0),(35, 1000, 0),(36, 1000, 0),(37, 1000, 0),(38, 1000, 0),(39, 1000, 0),(40, 1000, 0),(41, 1000, 0),(42, 1000, 0),(43, 1000, 0),(44, 1000, 0),(45, 1000, 0),(46, 1000, 0),(47, 1000, 0),(48, 1000, 0),(49, 1000, 0),(50, 1000, 0),(51, 1000, 0),(52, 1000, 0),(53, 1000, 0),(54, 1000, 0),(55, 1000, 0),(56, 1000, 0),(57, 1000, 0),(58, 1000, 0),(59, 1000, 0),(60, 1000, 0),(61, 1000, 0),(62, 1000, 0),(63, 1000, 0),(64, 1000, 0),(65, 1000, 0),(66, 1000, 0),(67, 1000, 0),(68, 1000, 0),(69, 1000, 0),(70, 1000, 0),(71, 1000, 0),(72, 1000, 0),(73, 1000, 0),(74, 1000, 0),(75, 1000, 0),(76, 1000, 0),(77, 1000, 0),(78, 1000, 0),(79, 1000, 0),(80, 1000, 0),(81, 1000, 0),(82, 1000, 0),(83, 1000, 0),(84, 1000, 0),(85, 1000, 0),(86, 1000, 0),(87, 1000, 0),(88, 1000, 0),(89, 1000, 0),(90, 1000, 0),(91, 1000, 0),(92, 1000, 0),(93, 1000, 0),(94, 1000, 0),(95, 1000, 0),(96, 1000, 0),(97, 1000, 0),(98, 1000, 0),(99, 1000, 0)"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/changefeed_pause_resume/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... check diff failed 9-th time, retry later + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_fast_fail.cli.2665.out cli tso query --pd=http://127.0.0.1:2379 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489988793441397415 TiDB Cluster ID. tikv_gc_leader_uuid 65821e6f3b40005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-j-shst4, pid:3983, start at 2025-04-06 08:21:58.136656637 +0800 CST m=+2.774231980 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:23:58.147 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:21:58.125 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:11:58.125 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489988793441397415 TiDB Cluster ID. tikv_gc_leader_uuid 65821e6f3b40005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-j-shst4, pid:3983, start at 2025-04-06 08:21:58.136656637 +0800 CST m=+2.774231980 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:23:58.147 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:21:58.125 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:11:58.125 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489988798273632371 TiDB Cluster ID. tikv_gc_leader_uuid 65821e6f5440006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-j-shst4, pid:4054, start at 2025-04-06 08:21:58.236890338 +0800 CST m=+2.800401068 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:23:58.246 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:21:58.225 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:11:58.225 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/cdc_server_tips/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/cdc_server_tips/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/cdc_server_tips/tiflash-proxy.toml"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/cdc_server_tips/tiflash/log/proxy.log"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/cdc_server_tips/tiflash/db/proxy"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 06 Apr 2025 00:21:59 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/e684662d-022f-4a79-9ef3-0a59dba96193 {UpstreamID:7489988628380945915 Namespace:default ID:e684662d-022f-4a79-9ef3-0a59dba96193 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-06 08:21:28.032207553 +0800 CST StartTs:457152628763131907 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003f319e0 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457152630100066306} {CheckpointTs:457152636994191362 MinTableBarrierTs:457152638305173507 AdminJobType:noop} span: {tableID:116, startKey:7480000000000000ff745f720000000000fa, endKey:7480000000000000ff745f730000000000fa}, resolvedTs: 457152636994191362, checkpointTs: 457152636994191362, state: Preparing span: {tableID:112, startKey:7480000000000000ff705f720000000000fa, endKey:7480000000000000ff705f730000000000fa}, resolvedTs: 457152636994191362, checkpointTs: 457152636994191362, state: Preparing span: {tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}, resolvedTs: 457152636994191362, checkpointTs: 457152636994191362, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/05c1d93b-6d57-468c-9ac7-293d116cde80 {"id":"05c1d93b-6d57-468c-9ac7-293d116cde80","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898916} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319608790546ad 05c1d93b-6d57-468c-9ac7-293d116cde80 /tidb/cdc/default/default/changefeed/info/e684662d-022f-4a79-9ef3-0a59dba96193 {"upstream-id":7489988628380945915,"namespace":"default","changefeed-id":"e684662d-022f-4a79-9ef3-0a59dba96193","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-06T08:21:28.032207553+08:00","start-ts":457152628763131907,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457152630100066306} /tidb/cdc/default/default/changefeed/status/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":457152636994191362,"min-table-barrier-ts":457152638305173507,"admin-job-type":0} /tidb/cdc/default/default/task/position/05c1d93b-6d57-468c-9ac7-293d116cde80/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489988628380945915 {"id":7489988628380945915,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/e684662d-022f-4a79-9ef3-0a59dba96193 {UpstreamID:7489988628380945915 Namespace:default ID:e684662d-022f-4a79-9ef3-0a59dba96193 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-06 08:21:28.032207553 +0800 CST StartTs:457152628763131907 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003f319e0 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457152630100066306} {CheckpointTs:457152636994191362 MinTableBarrierTs:457152638305173507 AdminJobType:noop} span: {tableID:116, startKey:7480000000000000ff745f720000000000fa, endKey:7480000000000000ff745f730000000000fa}, resolvedTs: 457152636994191362, checkpointTs: 457152636994191362, state: Preparing span: {tableID:112, startKey:7480000000000000ff705f720000000000fa, endKey:7480000000000000ff705f730000000000fa}, resolvedTs: 457152636994191362, checkpointTs: 457152636994191362, state: Preparing span: {tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}, resolvedTs: 457152636994191362, checkpointTs: 457152636994191362, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/05c1d93b-6d57-468c-9ac7-293d116cde80 {"id":"05c1d93b-6d57-468c-9ac7-293d116cde80","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898916} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319608790546ad 05c1d93b-6d57-468c-9ac7-293d116cde80 /tidb/cdc/default/default/changefeed/info/e684662d-022f-4a79-9ef3-0a59dba96193 {"upstream-id":7489988628380945915,"namespace":"default","changefeed-id":"e684662d-022f-4a79-9ef3-0a59dba96193","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-06T08:21:28.032207553+08:00","start-ts":457152628763131907,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457152630100066306} /tidb/cdc/default/default/changefeed/status/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":457152636994191362,"min-table-barrier-ts":457152638305173507,"admin-job-type":0} /tid+ grep -q 'failed to get info:' b/cdc/default/default/task/position/05c1d93b-6d57-468c-9ac7-293d116cde80/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489988628380945915 {"id":7489988628380945915,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/e684662d-022f-4a79-9ef3-0a59dba96193 {UpstreamID:7489988628380945915 Namespace:default ID:e684662d-022f-4a79-9ef3-0a59dba96193 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-06 08:21:28.032207553 +0800 CST StartTs:457152628763131907 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003f319e0 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457152630100066306} {CheckpointTs:457152636994191362 MinTableBarrierTs:457152638305173507 AdminJobType:noop} span: {tableID:116, startKey:7480000000000000ff745f720000000000fa, endKey:7480000000000000ff745f730000000000fa}, resolvedTs: 457152636994191362, checkpointTs: 457152636994191362, state: Preparing span: {tableID:112, startKey:7480000000000000ff705f720000000000fa, endKey:7480000000000000ff705f730000000000fa}, resolvedTs: 457152636994191362, checkpointTs: 457152636994191362, state: Preparing span: {tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}, resolvedTs: 457152636994191362, checkpointTs: 457152636994191362, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/05c1d93b-6d57-468c-9ac7-293d116cde80 {"id":"05c1d93b-6d57-468c-9ac7-293d116cde80","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898916} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319608790546ad 05c1d93b-6d57-468c-9ac7-293d116cde80 /tidb/cdc/default/default/changefeed/info/e684662d-022f-4a79-9ef3-0a59dba96193 {"upstream-id":7489988628380945915,"namespace":"default","changefeed-id":"e684662d-022f-4a79-9ef3-0a59dba96193","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-06T08:21:28.032207553+08:00","start-ts":457152628763131907,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457152630100066306} /tidb/cdc/default/default/changefeed/status/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":457152636994191362,"min-table-barrier-ts":457152638305173507,"admin-job-type":0} /tidb/cdc/default/default/task/position/05c1d93b-6d57-468c-9ac7-293d116cde80/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489988628380945915 {"id":7489988628380945915,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x cdc.test cli capture list 2>&1 | grep '"is-owner": true' start tidb cluster in /tmp/tidb_cdc_test/ddl_sequence Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489988795949182453 TiDB Cluster ID. tikv_gc_leader_uuid 65821e6f1040004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-8-31x9h, pid:7522, start at 2025-04-06 08:21:57.955205036 +0800 CST m=+2.205745436 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:23:57.962 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:21:57.953 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:11:57.953 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489988795949182453 TiDB Cluster ID. tikv_gc_leader_uuid 65821e6f1040004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-8-31x9h, pid:7522, start at 2025-04-06 08:21:57.955205036 +0800 CST m=+2.205745436 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:23:57.962 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:21:57.953 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:11:57.953 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489988795035962242 TiDB Cluster ID. tikv_gc_leader_uuid 65821e6eefc0003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-8-31x9h, pid:7602, start at 2025-04-06 08:21:57.826349653 +0800 CST m=+1.990349043 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:23:57.833 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:21:57.823 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:11:57.823 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/api_v2/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/api_v2/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/api_v2/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/api_v2/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/api_v2/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table ddl_manager.finish_mark not exists for 48-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489988782819250812 TiDB Cluster ID. tikv_gc_leader_uuid 65821e6f8dc0006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-x-4tfw6, pid:3932, start at 2025-04-06 08:21:58.461970513 +0800 CST m=+4.421167207 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:23:58.470 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:21:58.455 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:11:58.455 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489988782819250812 TiDB Cluster ID. tikv_gc_leader_uuid 65821e6f8dc0006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-x-4tfw6, pid:3932, start at 2025-04-06 08:21:58.461970513 +0800 CST m=+4.421167207 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:23:58.470 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:21:58.455 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:11:58.455 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489988789204197783 TiDB Cluster ID. tikv_gc_leader_uuid 65821e6f4040007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-x-4tfw6, pid:4004, start at 2025-04-06 08:21:58.155639835 +0800 CST m=+4.060844188 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:23:58.164 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:21:58.145 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:11:58.145 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/region_merge/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/region_merge/tiflash/log/error.log arg matches is ArgMatches { args: {"config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/region_merge/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/region_merge/tiflash/log/proxy.log"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/region_merge/tiflash/db/proxy"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Verifying downstream PD is started... [2025/04/06 08:22:01.092 +08:00] [INFO] [case.go:687] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark] "is-owner": true, run task successfully check diff failed 10-th time, retry later + set +x + tso='457152638273454081 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 457152638273454081 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Sun Apr 6 08:22:01 CST 2025] <<<<<< START cdc server in changefeed_fast_fail case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/InjectChangefeedFastFailError=return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_fast_fail.27142716.out server --log-file /tmp/tidb_cdc_test/changefeed_fast_fail/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_fast_fail/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/batch_add_table/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... Starting Upstream TiKV... start tidb cluster in /tmp/tidb_cdc_test/changefeed_pause_resume Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table ddl_manager.finish_mark not exists for 49-th check, retry later + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cdc_server_tips.cli.5299.out cli tso query --pd=http://127.0.0.1:2379 [Sun Apr 6 08:22:02 CST 2025] <<<<<< START cdc server in region_merge case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.region_merge.53145316.out server --log-file /tmp/tidb_cdc_test/region_merge/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/region_merge/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 owner pid: 3357 owner id 05c1d93b-6d57-468c-9ac7-293d116cde80 process status: T ETCDCTL_API=3 etcdctl get /tidb/cdc/default/__cdc_meta__/owner --prefix | grep -v '05c1d93b-6d57-468c-9ac7-293d116cde80' /tidb/cdc/default/__cdc_meta__/owner/22319608790546ad run task successfully process status: S cdc.test cli capture list 2>&1 | grep '"is-owner": true' Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... [Sun Apr 6 08:22:02 CST 2025] <<<<<< START cdc server in api_v2 case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.api_v2.89898991.out server --log-file /tmp/tidb_cdc_test/api_v2/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/api_v2/cdc_data --cluster-id default + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 build mysql_test successfully run all mysql test cases time="2025-04-06T08:21:59+08:00" level=info msg="running tests: [alter_table bigint composite_index concurrent_ddl date_formats drop mysql_replace]" time="2025-04-06T08:21:59+08:00" level=warning msg="Create new db&{{{} {} 0} 0xc0001ac000 {{} {} 0} {0 0} [0xc000308000] {[]} 1 0xc0001b0000 false map[0xc000308000:map[0xc000308000:true]] map[] 0 0 0 0 0 0 0 0 0x4a1c00}" Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 11-th time, retry later [2025/04/06 08:22:03.099 +08:00] [INFO] [case.go:687] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark] + set +x + tso='457152639040487426 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 457152639040487426 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore table ddl_manager.finish_mark not exists for 50-th check, retry later try a VALID cdc server command [Sun Apr 6 08:22:04 CST 2025] <<<<<< START cdc server in cdc_server_tips case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cdc_server_tips.53475349.out server --log-file /tmp/tidb_cdc_test/cdc_server_tips/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/cdc_server_tips/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/processor_etcd_worker_delay/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) "is-owner": true, run task successfully test_expire_owner pass wait process cdc.test exit for 1-th time... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 06 Apr 2025 00:22:04 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5ca3c71e-2654-4172-82a3-009d25ab2990 {"id":"5ca3c71e-2654-4172-82a3-009d25ab2990","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898922} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/2231960879908ef0 5ca3c71e-2654-4172-82a3-009d25ab2990 /tidb/cdc/default/default/upstream/7489988783608874346 {"id":7489988783608874346,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5ca3c71e-2654-4172-82a3-009d25ab2990 {"id":"5ca3c71e-2654-4172-82a3-009d25ab2990","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898922} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/2231960879908ef0 5ca3c71e-2654-4172-82a3-009d25ab2990 /tidb/cdc/default/default/upstream/7489988783608874346 {"id":7489988783608874346,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5ca3c71e-2654-4172-82a3-009d25ab2990 {"id":"5ca3c71e-2654-4172-82a3-009d25ab2990","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898922} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/2231960879908ef0 5ca3c71e-2654-4172-82a3-009d25ab2990 /tidb/cdc/default/default/upstream/7489988783608874346 {"id":7489988783608874346,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_fast_fail.cli.2766.out cli changefeed create --start-ts=457152638273454081 '--sink-uri=mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1' -c changefeed-fast-fail ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Create changefeed successfully! ID: changefeed-fast-fail Info: {"upstream_id":7489988783608874346,"namespace":"default","id":"changefeed-fast-fail","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create_time":"2025-04-06T08:22:04.836216679+08:00","start_ts":457152638273454081,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457152638273454081,"checkpoint_ts":457152638273454081,"checkpoint_time":"2025-04-06 08:21:59.195"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... check diff failed 12-th time, retry later start tidb cluster in /tmp/tidb_cdc_test/batch_add_table Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 06 Apr 2025 00:22:05 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f1ed0803-9c8d-4871-b4bf-870e91dc5c14 {"id":"f1ed0803-9c8d-4871-b4bf-870e91dc5c14","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898922} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319608799fb8e0 f1ed0803-9c8d-4871-b4bf-870e91dc5c14 /tidb/cdc/default/default/upstream/7489988795949182453 {"id":7489988795949182453,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f1ed0803-9c8d-4871-b4bf-870e91dc5c14 {"id":"f1ed0803-9c8d-4871-b4bf-870e91dc5c14","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898922} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319608799fb8e0 f1ed0803-9c8d-4871-b4bf-870e91dc5c14 /tidb/cdc/default/default/upstream/7489988795949182453 {"id":7489988795949182453,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f1ed0803-9c8d-4871-b4bf-870e91dc5c14 {"id":"f1ed0803-9c8d-4871-b4bf-870e91dc5c14","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898922} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319608799fb8e0 f1ed0803-9c8d-4871-b4bf-870e91dc5c14 /tidb/cdc/default/default/upstream/7489988795949182453 {"id":7489988795949182453,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x wait process cdc.test exit for 2-th time... [2025/04/06 08:22:05.106 +08:00] [INFO] [case.go:687] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark] Create changefeed successfully! ID: cf-blackhole Info: {"upstream_id":7489988795949182453,"namespace":"default","id":"cf-blackhole","sink_uri":"blackhole:","create_time":"2025-04-06T08:22:05.437544591+08:00","start_ts":457152639887736838,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457152639887736838,"checkpoint_ts":457152639887736838,"checkpoint_time":"2025-04-06 08:22:05.353"} + endpoints=http://127.0.0.1:2379 + changefeed_id=cf-blackhole + expected_state=normal + error_msg=null + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c cf-blackhole -s + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 06 Apr 2025 00:22:05 GMT < Content-Length: 864 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/62133df6-da49-451e-8e23-0d66632b6cae {"id":"62133df6-da49-451e-8e23-0d66632b6cae","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898922} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087998d5ed 62133df6-da49-451e-8e23-0d66632b6cae /tidb/cdc/default/default/upstream/7489988782819250812 {"id":7489988782819250812,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/62133df6-da49-451e-8e23-0d66632b6cae {"id":"62133df6-da49-451e-8e23-0d66632b6cae","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898922} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087998d5ed 62133df6-da49-451e-8e23-0d66632b6cae /tidb/cdc/default/default/upstream/7489988782819250812 {"id":7489988782819250812,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/62133df6-da49-451e-8e23-0d66632b6cae {"id":"62133df6-da49-451e-8e23-0d66632b6cae","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898922} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087998d5ed 62133df6-da49-451e-8e23-0d66632b6cae /tidb/cdc/default/default/upstream/7489988782819250812 {"id":7489988782819250812,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: 2d9c8be9-fc7e-4c27-8b94-81ee9c176bf7 Info: {"upstream_id":7489988782819250812,"namespace":"default","id":"2d9c8be9-fc7e-4c27-8b94-81ee9c176bf7","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-06T08:22:05.598947396+08:00","start_ts":457152639927582724,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457152639927582724,"checkpoint_ts":457152639927582724,"checkpoint_time":"2025-04-06 08:22:05.505"} + info='{ "upstream_id": 7489988795949182453, "namespace": "default", "id": "cf-blackhole", "state": "normal", "checkpoint_tso": 457152639887736838, "checkpoint_time": "2025-04-06 08:22:05.353", "error": null }' + echo '{ "upstream_id": 7489988795949182453, "namespace": "default", "id": "cf-blackhole", "state": "normal", "checkpoint_tso": 457152639887736838, "checkpoint_time": "2025-04-06 08:22:05.353", "error": null }' { "upstream_id": 7489988795949182453, "namespace": "default", "id": "cf-blackhole", "state": "normal", "checkpoint_tso": 457152639887736838, "checkpoint_time": "2025-04-06 08:22:05.353", "error": null } ++ echo '{' '"upstream_id":' 7489988795949182453, '"namespace":' '"default",' '"id":' '"cf-blackhole",' '"state":' '"normal",' '"checkpoint_tso":' 457152639887736838, '"checkpoint_time":' '"2025-04-06' '08:22:05.353",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7489988795949182453, '"namespace":' '"default",' '"id":' '"cf-blackhole",' '"state":' '"normal",' '"checkpoint_tso":' 457152639887736838, '"checkpoint_time":' '"2025-04-06' '08:22:05.353",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] + GO111MODULE=on + go run main.go model.go request.go cases.go go: downloading go1.23.4 (linux/amd64) cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit run test case test_owner_cleanup_stale_tasks [Sun Apr 6 08:22:05 CST 2025] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.35283530.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_owner_cleanup_stale_tasks.server1.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_owner_cleanup_stale_tasks.server1 --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x check_changefeed_state http://127.0.0.1:2379 changefeed-fast-fail failed ErrStartTsBeforeGC + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-fast-fail + expected_state=failed + error_msg=ErrStartTsBeforeGC + tls_dir=ErrStartTsBeforeGC + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-fast-fail -s + info='{ "upstream_id": 7489988783608874346, "namespace": "default", "id": "changefeed-fast-fail", "state": "failed", "checkpoint_tso": 457152638273454081, "checkpoint_time": "2025-04-06 08:21:59.195", "error": { "time": "2025-04-06T08:22:04.921537089+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrStartTsBeforeGC", "message": "[CDC:ErrStartTsBeforeGC]InjectChangefeedFastFailError" } }' + echo '{ "upstream_id": 7489988783608874346, "namespace": "default", "id": "changefeed-fast-fail", "state": "failed", "checkpoint_tso": 457152638273454081, "checkpoint_time": "2025-04-06 08:21:59.195", "error": { "time": "2025-04-06T08:22:04.921537089+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrStartTsBeforeGC", "message": "[CDC:ErrStartTsBeforeGC]InjectChangefeedFastFailError" } }' { "upstream_id": 7489988783608874346, "namespace": "default", "id": "changefeed-fast-fail", "state": "failed", "checkpoint_tso": 457152638273454081, "checkpoint_time": "2025-04-06 08:21:59.195", "error": { "time": "2025-04-06T08:22:04.921537089+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrStartTsBeforeGC", "message": "[CDC:ErrStartTsBeforeGC]InjectChangefeedFastFailError" } } ++ echo '{' '"upstream_id":' 7489988783608874346, '"namespace":' '"default",' '"id":' '"changefeed-fast-fail",' '"state":' '"failed",' '"checkpoint_tso":' 457152638273454081, '"checkpoint_time":' '"2025-04-06' '08:21:59.195",' '"error":' '{' '"time":' '"2025-04-06T08:22:04.921537089+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrStartTsBeforeGC",' '"message":' '"[CDC:ErrStartTsBeforeGC]InjectChangefeedFastFailError"' '}' '}' ++ jq -r .state + state=failed + [[ ! failed == \f\a\i\l\e\d ]] ++ echo '{' '"upstream_id":' 7489988783608874346, '"namespace":' '"default",' '"id":' '"changefeed-fast-fail",' '"state":' '"failed",' '"checkpoint_tso":' 457152638273454081, '"checkpoint_time":' '"2025-04-06' '08:21:59.195",' '"error":' '{' '"time":' '"2025-04-06T08:22:04.921537089+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrStartTsBeforeGC",' '"message":' '"[CDC:ErrStartTsBeforeGC]InjectChangefeedFastFailError"' '}' '}' ++ jq -r .error.message + message='[CDC:ErrStartTsBeforeGC]InjectChangefeedFastFailError' + [[ ! [CDC:ErrStartTsBeforeGC]InjectChangefeedFastFailError =~ ErrStartTsBeforeGC ]] run task successfully table ddl_manager.finish_mark not exists for 51-th check, retry later Verifying downstream PD is started... split_and_random_merge scale: 20 *************************** 1. row *************************** count(distinct region_id): 1 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489988825213157151 TiDB Cluster ID. tikv_gc_leader_uuid 65821e75cfc0002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-6-zhp59, pid:4249, start at 2025-04-06 08:22:04.864116176 +0800 CST m=+2.102484448 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:24:04.874 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:22:04.863 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:12:04.863 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489988825213157151 TiDB Cluster ID. tikv_gc_leader_uuid 65821e75cfc0002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-6-zhp59, pid:4249, start at 2025-04-06 08:22:04.864116176 +0800 CST m=+2.102484448 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:24:04.874 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:22:04.863 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:12:04.863 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489988823535072049 TiDB Cluster ID. tikv_gc_leader_uuid 65821e75e70000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-6-zhp59, pid:4324, start at 2025-04-06 08:22:04.981224538 +0800 CST m=+2.142234792 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:24:04.989 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:22:04.956 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:12:04.956 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/consistent_replicate_gbk/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/consistent_replicate_gbk/tiflash/log/error.log arg matches is ArgMatches { args: {"addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/consistent_replicate_gbk/tiflash-proxy.toml"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/consistent_replicate_gbk/tiflash/db/proxy"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/consistent_replicate_gbk/tiflash/log/proxy.log"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [2025/04/06 08:22:07.113 +08:00] [INFO] [case.go:687] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 06 Apr 2025 00:22:07 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b3b51c29-c0f4-4829-8c46-4666ad04a4bd {"id":"b3b51c29-c0f4-4829-8c46-4666ad04a4bd","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898924} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/2231960879a1eff2 b3b51c29-c0f4-4829-8c46-4666ad04a4bd /tidb/cdc/default/default/upstream/7489988793441397415 {"id":7489988793441397415,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b3b51c29-c0f4-4829-8c46-4666ad04a4bd {"id":"b3b51c29-c0f4-4829-8c46-4666ad04a4bd","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898924} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/2231960879a1eff2 b3b51c29-c0f4-4829-8c46-4666ad04a4bd /tidb/cdc/default/default/upstream/7489988793441397415 {"id":7489988793441397415,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b3b51c29-c0f4-4829-8c46-4666ad04a4bd {"id":"b3b51c29-c0f4-4829-8c46-4666ad04a4bd","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898924} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/2231960879a1eff2 b3b51c29-c0f4-4829-8c46-4666ad04a4bd /tidb/cdc/default/default/upstream/7489988793441397415 {"id":7489988793441397415,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x start tidb cluster in /tmp/tidb_cdc_test/processor_etcd_worker_delay Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... check diff failed 13-th time, retry later Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release wait process cdc.test exit for 1-th time... Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 2-th time... table ddl_manager.finish_mark not exists for 52-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489988830202534805 TiDB Cluster ID. tikv_gc_leader_uuid 65821e77f58000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-m-cm2pq, pid:7652, start at 2025-04-06 08:22:07.094000068 +0800 CST m=+2.977379520 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:24:07.140 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:22:07.130 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:12:07.130 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489988830202534805 TiDB Cluster ID. tikv_gc_leader_uuid 65821e77f58000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-m-cm2pq, pid:7652, start at 2025-04-06 08:22:07.094000068 +0800 CST m=+2.977379520 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:24:07.140 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:22:07.130 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:12:07.130 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489988829759391985 TiDB Cluster ID. tikv_gc_leader_uuid 65821e780c0000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-m-cm2pq, pid:7718, start at 2025-04-06 08:22:07.179862655 +0800 CST m=+2.999361066 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:24:07.187 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:22:07.152 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:12:07.152 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/ddl_sequence/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/ddl_sequence/tiflash/log/error.log arg matches is ArgMatches { args: {"config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/ddl_sequence/tiflash-proxy.toml"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/ddl_sequence/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/ddl_sequence/tiflash/db/proxy"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Verifying downstream PD is started... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Sun Apr 6 08:22:08 CST 2025] <<<<<< run test case changefeed_fast_fail success! >>>>>> + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 06 Apr 2025 00:22:08 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/e684662d-022f-4a79-9ef3-0a59dba96193 {UpstreamID:7489988628380945915 Namespace:default ID:e684662d-022f-4a79-9ef3-0a59dba96193 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-06 08:21:28.032207553 +0800 CST StartTs:457152628763131907 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00431b320 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457152630100066306} {CheckpointTs:457152639353749507 MinTableBarrierTs:457152640664207364 AdminJobType:noop} span: {tableID:112, startKey:7480000000000000ff705f720000000000fa, endKey:7480000000000000ff705f730000000000fa}, resolvedTs: 457152639353749507, checkpointTs: 457152639353749507, state: Preparing span: {tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}, resolvedTs: 457152639353749507, checkpointTs: 457152639353749507, state: Preparing span: {tableID:116, startKey:7480000000000000ff745f720000000000fa, endKey:7480000000000000ff745f730000000000fa}, resolvedTs: 457152639353749507, checkpointTs: 457152639353749507, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b6166d19-012a-453b-adb9-597ea8e2a51a {"id":"b6166d19-012a-453b-adb9-597ea8e2a51a","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898925} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087905471c b6166d19-012a-453b-adb9-597ea8e2a51a /tidb/cdc/default/default/changefeed/info/e684662d-022f-4a79-9ef3-0a59dba96193 {"upstream-id":7489988628380945915,"namespace":"default","changefeed-id":"e684662d-022f-4a79-9ef3-0a59dba96193","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-06T08:21:28.032207553+08:00","start-ts":457152628763131907,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457152630100066306} /tidb/cdc/default/default/changefeed/status/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":457152639353749507,"min-table-barrier-ts":457152640664207364,"admin-job-type":0} /tidb/cdc/default/default/task/position/b6166d19-012a-453b-adb9-597ea8e2a51a/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489988628380945915 {"id":7489988628380945915,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/e684662d-022f-4a79-9ef3-0a59dba96193 {UpstreamID:7489988628380945915 Namespace:default ID:e684662d-022f-4a79-9ef3-0a59dba96193 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-06 08:21:28.032207553 +0800 CST StartTs:457152628763131907 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00431b320 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457152630100066306} {CheckpointTs:457152639353749507 MinTableBarrierTs:457152640664207364 AdminJobType:noop} span: {tableID:112, startKey:7480000000000000ff705f720000000000fa, endKey:7480000000000000ff705f730000000000fa}, resolvedTs: 457152639353749507, checkpointTs: 457152639353749507, state: Preparing span: {tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}, resolvedTs: 457152639353749507, checkpointTs: 457152639353749507, state: Preparing span: {tableID:116, startKey:7480000000000000ff745f720000000000fa, endKey:7480000000000000ff745f730000000000fa}, resolvedTs: 457152639353749507, checkpointTs: 457152639353749507, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b6166d19-012a-453b-adb9-597ea8e2a51a {"id":"b6166d19-012a-453b-adb9-597ea8e2a51a","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898925} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087905471c b6166d19-012a-453b-adb9-597ea8e2a51a /tidb/cdc/default/default/changefeed/info/e684662d-022f-4a79-9ef3-0a59dba96193 {"upstream-id":7489988628380945915,"namespace":"default","changefeed-id":"e684662d-022f-4a79-9ef3-0a59dba96193","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-06T08:21:28.032207553+08:00","start-ts":457152628763131907,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457152630100066306} /tidb/cdc/default/default/changefeed/status/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":457152639353749507,"min-table-barrier-ts":457152640664207364,"admin-job-type":0} /tid+ grep -q 'failed to get info:' b/cdc/default/default/task/position/b6166d19-012a-453b-adb9-597ea8e2a51a/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489988628380945915 {"id":7489988628380945915,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/e684662d-022f-4a79-9ef3-0a59dba96193 {UpstreamID:7489988628380945915 Namespace:default ID:e684662d-022f-4a79-9ef3-0a59dba96193 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-06 08:21:28.032207553 +0800 CST StartTs:457152628763131907 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00431b320 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457152630100066306} {CheckpointTs:457152639353749507 MinTableBarrierTs:457152640664207364 AdminJobType:noop} span: {tableID:112, startKey:7480000000000000ff705f720000000000fa, endKey:7480000000000000ff705f730000000000fa}, resolvedTs: 457152639353749507, checkpointTs: 457152639353749507, state: Preparing span: {tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}, resolvedTs: 457152639353749507, checkpointTs: 457152639353749507, state: Preparing span: {tableID:116, startKey:7480000000000000ff745f720000000000fa, endKey:7480000000000000ff745f730000000000fa}, resolvedTs: 457152639353749507, checkpointTs: 457152639353749507, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b6166d19-012a-453b-adb9-597ea8e2a51a {"id":"b6166d19-012a-453b-adb9-597ea8e2a51a","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898925} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087905471c b6166d19-012a-453b-adb9-597ea8e2a51a /tidb/cdc/default/default/changefeed/info/e684662d-022f-4a79-9ef3-0a59dba96193 {"upstream-id":7489988628380945915,"namespace":"default","changefeed-id":"e684662d-022f-4a79-9ef3-0a59dba96193","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-06T08:21:28.032207553+08:00","start-ts":457152628763131907,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457152630100066306} /tidb/cdc/default/default/changefeed/status/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":457152639353749507,"min-table-barrier-ts":457152640664207364,"admin-job-type":0} /tidb/cdc/default/default/task/position/b6166d19-012a-453b-adb9-597ea8e2a51a/e684662d-022f-4a79-9ef3-0a59dba96193 + grep -q 'etcd info' {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489988628380945915 {"id":7489988628380945915,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list 2>&1 | grep '"is-owner": true' [Sun Apr 6 08:22:09 CST 2025] <<<<<< START cdc server in consistent_replicate_gbk case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.consistent_replicate_gbk.56905692.out server --log-file /tmp/tidb_cdc_test/consistent_replicate_gbk/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/consistent_replicate_gbk/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2025/04/06 08:22:09.120 +08:00] [INFO] [case.go:687] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark] check diff failed 14-th time, retry later Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 53-th check, retry later "is-owner": true, run task successfully ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_sequence.cli.9119.out cli tso query --pd=http://127.0.0.1:2379 go: downloading github.com/pingcap/log v1.1.1-0.20241212030209-7e3ff8601a2a go: downloading github.com/pingcap/errors v0.11.5-0.20240318064555-6bd07397691f go: downloading go.uber.org/zap v1.27.0 go: downloading google.golang.org/grpc v1.64.0 go: downloading github.com/pingcap/failpoint v0.0.0-20240528011301-b51a646c7c86 go: downloading github.com/tikv/pd/client v0.0.0-20241111073742-238d4d79ea31 go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20241223052309-3735ed55a394 go: downloading github.com/coreos/go-semver v0.3.1 go: downloading github.com/gin-gonic/gin v1.9.1 go: downloading github.com/pingcap/kvproto v0.0.0-20241120022153-92b0414aeed8 go: downloading github.com/xdg/scram v1.0.5 go: downloading github.com/IBM/sarama v1.41.2 go: downloading github.com/prometheus/client_golang v1.20.5 go: downloading github.com/google/uuid v1.6.0 go: downloading go.etcd.io/etcd/server/v3 v3.5.12 go: downloading golang.org/x/sync v0.10.0 go: downloading golang.org/x/time v0.7.0 go: downloading go.etcd.io/etcd/client/v3 v3.5.12 go: downloading github.com/go-sql-driver/mysql v1.7.1 go: downloading github.com/tinylib/msgp v1.1.6 go: downloading github.com/tikv/client-go/v2 v2.0.8-0.20241209094930-06d7f4b9233b go: downloading go.uber.org/multierr v1.11.0 go: downloading go.uber.org/atomic v1.11.0 go: downloading github.com/apache/pulsar-client-go v0.11.0 go: downloading github.com/cockroachdb/pebble v1.1.0 go: downloading github.com/pingcap/tidb/pkg/parser v0.0.0-20241223052309-3735ed55a394 go: downloading github.com/aws/aws-sdk-go-v2 v1.19.1 go: downloading gopkg.in/natefinch/lumberjack.v2 v2.2.1 go: downloading github.com/klauspost/compress v1.17.9 go: downloading github.com/pierrec/lz4/v4 v4.1.18 go: downloading cloud.google.com/go/storage v1.39.1 go: downloading github.com/KimMachineGun/automemlimit v0.2.4 go: downloading github.com/aws/aws-sdk-go v1.55.5 go: downloading github.com/json-iterator/go v1.1.12 go: downloading github.com/shirou/gopsutil/v3 v3.24.5 go: downloading github.com/benbjohnson/clock v1.1.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v1.16.0 go: downloading github.com/pingcap/tidb-dashboard v0.0.0-20240326110213-9768844ff5d7 go: downloading github.com/uber-go/atomic v1.4.0 go: downloading github.com/goccy/go-json v0.10.2 go: downloading github.com/stretchr/testify v1.10.0 go: downloading go.etcd.io/etcd/api/v3 v3.5.12 go: downloading github.com/gogo/protobuf v1.3.2 go: downloading go.etcd.io/etcd/pkg/v3 v3.5.12 go: downloading github.com/cenkalti/backoff/v4 v4.2.1 go: downloading github.com/google/btree v1.1.2 go: downloading github.com/DATA-DOG/go-sqlmock v1.5.0 go: downloading github.com/imdario/mergo v0.3.16 go: downloading github.com/tikv/pd v1.1.0-beta.0.20240407022249-7179657d129b go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.12 go: downloading cloud.google.com/go v0.112.2 go: downloading github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 [2025/04/06 08:22:11.128 +08:00] [INFO] [case.go:687] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark] go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 go: downloading github.com/containerd/cgroups v1.0.4 go: downloading golang.org/x/sys v0.28.0 go: downloading github.com/gavv/monotime v0.0.0-20190418164738-30dba4353424 go: downloading github.com/xdg/stringprep v1.0.3 go: downloading golang.org/x/crypto v0.31.0 go: downloading github.com/golang/protobuf v1.5.4 go: downloading google.golang.org/genproto/googleapis/api v0.0.0-20240401170217-c3f982113cda check diff failed 15-th time, retry later go: downloading github.com/go-mysql-org/go-mysql v1.9.1 go: downloading github.com/pingcap/check v0.0.0-20211026125417-57bd13f7b5f0 go: downloading github.com/philhofer/fwd v1.1.1 go: downloading github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc go: downloading github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 go: downloading google.golang.org/genproto v0.0.0-20240401170217-c3f982113cda go: downloading github.com/stretchr/objx v0.5.2 go: downloading github.com/gin-contrib/sse v0.1.0 go: downloading github.com/mattn/go-isatty v0.0.20 go: downloading golang.org/x/net v0.33.0 go: downloading github.com/go-playground/validator/v10 v10.14.0 go: downloading github.com/pelletier/go-toml/v2 v2.0.8 go: downloading github.com/ugorji/go/codec v1.2.11 go: downloading google.golang.org/protobuf v1.35.2 owner pid: 3533 owner id b6166d19-012a-453b-adb9-597ea8e2a51a [Sun Apr 6 08:22:11 CST 2025] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.36393641.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_owner_cleanup_stale_tasks.server2.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_owner_cleanup_stale_tasks.server2 --cluster-id default --addr 127.0.0.1:8301 ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd go: downloading github.com/coreos/go-systemd/v22 v22.5.0 go: downloading github.com/docker/go-units v0.5.0 go: downloading gorm.io/gorm v1.25.11 go: downloading github.com/beorn7/perks v1.0.1 go: downloading github.com/cespare/xxhash/v2 v2.3.0 go: downloading github.com/prometheus/client_model v0.6.1 go: downloading github.com/prometheus/common v0.61.0 go: downloading github.com/prometheus/procfs v0.15.1 go: downloading github.com/aws/smithy-go v1.13.5 go: downloading github.com/bits-and-blooms/bitset v1.14.3 go: downloading github.com/linkedin/goavro/v2 v2.11.1 go: downloading github.com/pkg/errors v0.9.1 go: downloading github.com/sirupsen/logrus v1.9.3 go: downloading github.com/eapache/go-resiliency v1.4.0 go: downloading github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 go: downloading github.com/eapache/queue v1.1.0 go: downloading github.com/hashicorp/go-multierror v1.1.1 go: downloading github.com/jcmturner/gofork v1.7.6 go: downloading github.com/jcmturner/gokrb5/v8 v8.4.4 go: downloading github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 go: downloading github.com/opentracing/opentracing-go v1.2.0 go: downloading github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a go: downloading github.com/robfig/cron v1.2.0 go: downloading github.com/hashicorp/golang-lru v0.5.1 go: downloading golang.org/x/oauth2 v0.24.0 table ddl_manager.finish_mark not exists for 54-th check, retry later Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489988849654225577 TiDB Cluster ID. tikv_gc_leader_uuid 65821e7af38000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-c-g3td9, pid:4470, start at 2025-04-06 08:22:10.151404497 +0800 CST m=+2.226119879 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:24:10.160 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:22:10.126 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:12:10.126 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489988849654225577 TiDB Cluster ID. tikv_gc_leader_uuid 65821e7af38000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-c-g3td9, pid:4470, start at 2025-04-06 08:22:10.151404497 +0800 CST m=+2.226119879 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:24:10.160 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:22:10.126 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:12:10.126 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489988841833557639 TiDB Cluster ID. tikv_gc_leader_uuid 65821e7b0e0000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-c-g3td9, pid:4547, start at 2025-04-06 08:22:10.280106711 +0800 CST m=+2.291592196 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:24:10.288 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:22:10.282 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:12:10.282 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/changefeed_pause_resume/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/changefeed_pause_resume/tiflash/log/error.log arg matches is ArgMatches { args: {"pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/changefeed_pause_resume/tiflash/log/proxy.log"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/changefeed_pause_resume/tiflash/db/proxy"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/changefeed_pause_resume/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 06 Apr 2025 00:22:12 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4af3131b-1520-4194-9759-6c8689c27f4b {"id":"4af3131b-1520-4194-9759-6c8689c27f4b","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898929} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/2231960879b924d8 4af3131b-1520-4194-9759-6c8689c27f4b /tidb/cdc/default/default/upstream/7489988825213157151 {"id":7489988825213157151,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4af3131b-1520-4194-9759-6c8689c27f4b {"id":"4af3131b-1520-4194-9759-6c8689c27f4b","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898929} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/2231960879b924d8 4af3131b-1520-4194-9759-6c8689c27f4b /tidb/cdc/default/default/upstream/7489988825213157151 {"id":7489988825213157151,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4af3131b-1520-4194-9759-6c8689c27f4b {"id":"4af3131b-1520-4194-9759-6c8689c27f4b","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898929} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/2231960879b924d8 4af3131b-1520-4194-9759-6c8689c27f4b /tidb/cdc/default/default/upstream/7489988825213157151 {"id":7489988825213157151,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + set +x + tso='457152641331888132 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 457152641331888132 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Sun Apr 6 08:22:12 CST 2025] <<<<<< START cdc server in ddl_sequence case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + GO_FAILPOINTS= + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_sequence.91609162.out server --log-file /tmp/tidb_cdc_test/ddl_sequence/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_sequence/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 go: downloading github.com/segmentio/kafka-go v0.4.41-0.20230526171612-f057b1d369cd go: downloading golang.org/x/text v0.21.0 go: downloading cloud.google.com/go/compute/metadata v0.3.0 go: downloading cloud.google.com/go/iam v1.1.7 go: downloading github.com/googleapis/gax-go/v2 v2.12.3 go: downloading google.golang.org/api v0.170.0 go: downloading github.com/joomcode/errorx v1.0.1 go: downloading github.com/cilium/ebpf v0.4.0 go: downloading github.com/godbus/dbus/v5 v5.0.4 go: downloading github.com/opencontainers/runtime-spec v1.0.2 go: downloading github.com/grpc-ecosystem/grpc-gateway v1.16.0 go: downloading github.com/soheilhy/cmux v0.1.5 go: downloading github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75 go: downloading github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 go: downloading go.etcd.io/bbolt v1.3.9 go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 go: downloading go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.22.0 go: downloading go.opentelemetry.io/otel v1.24.0 go: downloading go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.22.0 go: downloading go.opentelemetry.io/otel/sdk v1.24.0 go: downloading github.com/dustin/go-humanize v1.0.1 go: downloading sigs.k8s.io/yaml v1.4.0 go: downloading github.com/BurntSushi/toml v1.4.1-0.20240526193622-a339e1f7089c go: downloading github.com/Masterminds/semver v1.5.0 go: downloading github.com/siddontang/go-log v0.0.0-20180807004314-8d05993dda07 go: downloading github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726 go: downloading github.com/gabriel-vasile/mimetype v1.4.2 go: downloading github.com/go-playground/universal-translator v0.18.1 go: downloading github.com/leodido/go-urn v1.2.4 go: downloading github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5 go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 go: downloading github.com/shopspring/decimal v1.3.0 go: downloading google.golang.org/genproto/googleapis/rpc v0.0.0-20240515191416-fc5f0ca64291 go: downloading github.com/edwingeng/deque v0.0.0-20191220032131-8596380dee17 go: downloading github.com/jinzhu/now v1.1.5 go: downloading github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 go: downloading github.com/AthenZ/athenz v1.10.39 go: downloading github.com/spaolacci/murmur3 v1.1.0 go: downloading golang.org/x/mod v0.22.0 go: downloading github.com/DataDog/zstd v1.5.5 go: downloading github.com/pierrec/lz4 v2.6.1+incompatible go: downloading github.com/hashicorp/errwrap v1.0.0 go: downloading github.com/golang/snappy v0.0.4 go: downloading github.com/jcmturner/dnsutils/v2 v2.0.0 go: downloading github.com/hashicorp/go-uuid v1.0.3 go: downloading github.com/dgryski/go-farm v0.0.0-20240924180020-3414d57e47da go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 [2025/04/06 08:22:13.134 +08:00] [INFO] [case.go:687] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark] go: downloading github.com/twmb/murmur3 v1.1.6 go: downloading github.com/cloudfoundry/gosigar v1.3.6 go: downloading golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 go: downloading github.com/cockroachdb/errors v1.11.1 *************************** 1. row *************************** count(distinct region_id): 20 check diff failed 16-th time, retry later go: downloading github.com/cockroachdb/redact v1.1.5 go: downloading github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 go: downloading go.opencensus.io v0.23.1-0.20220331163232-052120675fac go: downloading go.opentelemetry.io/otel/trace v1.24.0 go: downloading github.com/xdg-go/scram v1.1.2 go: downloading github.com/gorilla/websocket v1.5.1 go: downloading github.com/spf13/pflag v1.0.5 go: downloading go.etcd.io/etcd/raft/v3 v3.5.12 go: downloading github.com/xiang90/probing v0.0.0-20221125231312-a49e3df8f510 go: downloading github.com/jonboulle/clockwork v0.4.0 go: downloading go.opentelemetry.io/otel/metric v1.24.0 table ddl_manager.finish_mark not exists for 55-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) go: downloading github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 go: downloading go.opentelemetry.io/proto/otlp v1.1.0 [Sun Apr 6 08:22:14 CST 2025] <<<<<< START cdc server in changefeed_pause_resume case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' go: downloading github.com/go-playground/locales v0.14.1 go: downloading github.com/jinzhu/inflection v1.0.0 + '[' -z '' ']' go: downloading github.com/golang-jwt/jwt v3.2.2+incompatible go: downloading github.com/99designs/keyring v1.2.1 + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) go: downloading github.com/jcmturner/rpc/v2 v2.0.3 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec go: downloading github.com/aws/aws-sdk-go-v2/config v1.18.30 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_pause_resume.58625864.out server --log-file /tmp/tidb_cdc_test/changefeed_pause_resume/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_pause_resume/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i <= 50 )) go: downloading github.com/aws/aws-sdk-go-v2/credentials v1.13.29 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused go: downloading github.com/aws/aws-sdk-go-v2/service/glue v1.58.1 * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table consistent_replicate_gbk.GBKTABLE not exists for 1-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489988850765574355 TiDB Cluster ID. tikv_gc_leader_uuid 65821e7ca78000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-x-hbn5c, pid:4011, start at 2025-04-06 08:22:11.886772344 +0800 CST m=+2.134592392 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:24:11.894 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:22:11.921 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:12:11.921 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489988850765574355 TiDB Cluster ID. tikv_gc_leader_uuid 65821e7ca78000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-x-hbn5c, pid:4011, start at 2025-04-06 08:22:11.886772344 +0800 CST m=+2.134592392 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:24:11.894 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:22:11.921 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:12:11.921 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489988856477013479 TiDB Cluster ID. tikv_gc_leader_uuid 65821e7cd8c000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-x-hbn5c, pid:4083, start at 2025-04-06 08:22:12.093953507 +0800 CST m=+2.258292274 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:24:12.101 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:22:12.067 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:12:12.067 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/batch_add_table/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/batch_add_table/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/batch_add_table/tiflash/db/proxy"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/batch_add_table/tiflash/log/proxy.log"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/batch_add_table/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } go: downloading github.com/jarcoal/httpmock v1.2.0 go: downloading github.com/mailru/easyjson v0.7.7 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 06 Apr 2025 00:22:14 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/e684662d-022f-4a79-9ef3-0a59dba96193 {UpstreamID:7489988628380945915 Namespace:default ID:e684662d-022f-4a79-9ef3-0a59dba96193 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-06 08:21:28.032207553 +0800 CST StartTs:457152628763131907 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003b2e990 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457152630100066306} {CheckpointTs:457152642237071367 MinTableBarrierTs:457152642237071367 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1e510708-5e63-4f24-a01f-9ef37aef0e9d {"id":"1e510708-5e63-4f24-a01f-9ef37aef0e9d","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898932} /tidb/cdc/default/__cdc_meta__/capture/b6166d19-012a-453b-adb9-597ea8e2a51a {"id":"b6166d19-012a-453b-adb9-597ea8e2a51a","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898925} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087905471c b6166d19-012a-453b-adb9-597ea8e2a51a /tidb/cdc/default/__cdc_meta__/owner/2231960879054765 1e510708-5e63-4f24-a01f-9ef37aef0e9d /tidb/cdc/default/default/changefeed/info/e684662d-022f-4a79-9ef3-0a59dba96193 {"upstream-id":7489988628380945915,"namespace":"default","changefeed-id":"e684662d-022f-4a79-9ef3-0a59dba96193","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-06T08:21:28.032207553+08:00","start-ts":457152628763131907,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457152630100066306} /tidb/cdc/default/default/changefeed/status/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":457152642237071367,"min-table-barrier-ts":457152642237071367,"admin-job-type":0} /tidb/cdc/default/default/task/position/1e510708-5e63-4f24-a01f-9ef37aef0e9d/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/b6166d19-012a-453b-adb9-597ea8e2a51a/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489988628380945915 {"id":7489988628380945915,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/e684662d-022f-4a79-9ef3-0a59dba96193 {UpstreamID:7489988628380945915 Namespace:default ID:e684662d-022f-4a79-9ef3-0a59dba96193 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-06 08:21:28.032207553 +0800 CST StartTs:457152628763131907 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003b2e990 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457152630100066306} {CheckpointTs:457152642237071367 MinTableBarrierTs:457152642237071367 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1e510708-5e63-4f24-a01f-9ef37aef0e9d {"id":"1e510708-5e63-4f24-a01f-9ef37aef0e9d","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898932} /tidb/cdc/default/__cdc_meta__/capture/b6166d19-012a-453b-adb9-597ea8e2a51a {"id":"b6166d19-012a-453b-adb9-597ea8e2a51a","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898925} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087905471c b6166d19-012a-453b-adb9-597ea8e2a51a /tidb/cdc/default/__cdc_meta__/owner/2231960879054765 1e510708-5e63-4f24-a01f-9ef37aef0e9d /tidb/cdc/default/default/changefeed/info/e684662d-022f-4a79-9ef3-0a59dba96193 {"upstream-id":7489988628380945915,"namespace":"default","changefeed-id":"e684662d-022f-4a79-9ef3-0a59dba96193","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-06T08:21:28.032207553+08:00","start-ts":457152628763131907,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457152630100066306} /tidb/cdc/default/default/changefeed/status/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":457152642237071367,"min-table-barrier-ts":457152642237071367,"admin-job-type":0} /tidb/cdc/default/default/task/position/1e510708-5e63-4f24-a01f-9ef37aef0e9d/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/b6166d19-012a-453b-adb9-597ea8e2a51a/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489988628380945915 {"id":7489988628380945915,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + grep -q 'etcd info' + echo ' *** processors info ***: changefeedID: default/e684662d-022f-4a79-9ef3-0a59dba96193 {UpstreamID:7489988628380945915 Namespace:default ID:e684662d-022f-4a79-9ef3-0a59dba96193 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-06 08:21:28.032207553 +0800 CST StartTs:457152628763131907 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003b2e990 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457152630100066306} {CheckpointTs:457152642237071367 MinTableBarrierTs:457152642237071367 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1e510708-5e63-4f24-a01f-9ef37aef0e9d {"id":"1e510708-5e63-4f24-a01f-9ef37aef0e9d","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898932} /tidb/cdc/default/__cdc_meta__/capture/b6166d19-012a-453b-adb9-597ea8e2a51a {"id":"b6166d19-012a-453b-adb9-597ea8e2a51a","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898925} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087905471c b6166d19-012a-453b-adb9-597ea8e2a51a /tidb/cdc/default/__cdc_meta__/owner/2231960879054765 1e510708-5e63-4f24-a01f-9ef37aef0e9d /tidb/cdc/default/default/changefeed/info/e684662d-022f-4a79-9ef3-0a59dba96193 {"upstream-id":7489988628380945915,"namespace":"default","changefeed-id":"e684662d-022f-4a79-9ef3-0a59dba96193","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-06T08:21:28.032207553+08:00","start-ts":457152628763131907,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457152630100066306} /tidb/cdc/default/default/changefeed/status/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":457152642237071367,"min-table-barrier-ts":457152642237071367,"admin-job-type":0} /tidb/cdc/default/default/task/position/1e510708-5e63-4f24-a01f-9ef37aef0e9d/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/b6166d19-012a-453b-adb9-597ea8e2a51a/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489988628380945915 {"id":7489988628380945915,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list --server 'http://127.0.0.1:8301' 2>&1 | grep -v "b6166d19-012a-453b-adb9-597ea8e2a51a" | grep id [2025/04/06 08:22:15.141 +08:00] [INFO] [case.go:687] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark] [2025/04/06 08:22:15.147 +08:00] [INFO] [case.go:697] ["try parse finishedTs from ticdc log"] [tableName=finishmark] /tmp/tidb_cdc_test/bank/cdcdown.log [2025/04/06 08:22:15.149 +08:00] [INFO] [case.go:717] ["total files"] [file="[/tmp/tidb_cdc_test/bank/cdcdown.log]"] found first match line: 457152642183069710 : [2025/04/06 08:22:14.914 +08:00] [INFO] [ddl_manager.go:298] ["handle a ddl job"] [namespace=default] [changefeed=changefeed-for-find-finished-ts] [tableID=134] [jobID=135] [query="CREATE TABLE IF NOT EXISTS `finishmark` (`foo` BIGINT PRIMARY KEY)"] [finishedTs=457152642183069710] [2025/04/06 08:22:15.158 +08:00] [INFO] [case.go:503] ["all tables synced"] [endTs=457152642183069710] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 06 Apr 2025 00:22:15 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/dbfa5af7-8c62-429c-aace-45df1d9614e7 {"id":"dbfa5af7-8c62-429c-aace-45df1d9614e7","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898932} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/2231960879c412df dbfa5af7-8c62-429c-aace-45df1d9614e7 /tidb/cdc/default/default/upstream/7489988830202534805 {"id":7489988830202534805,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/dbfa5af7-8c62-429c-aace-45df1d9614e7 {"id":"dbfa5af7-8c62-429c-aace-45df1d9614e7","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898932} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/2231960879c412df dbfa5af7-8c62-429c-aace-45df1d9614e7 /tidb/cdc/default/default/upstream/7489988830202534805 {"id":7489988830202534805,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/dbfa5af7-8c62-429c-aace-45df1d9614e7 {"id":"dbfa5af7-8c62-429c-aace-45df1d9614e7","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898932} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/2231960879c412df dbfa5af7-8c62-429c-aace-45df1d9614e7 /tidb/cdc/default/default/upstream/7489988830202534805 {"id":7489988830202534805,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_sequence.cli.9221.out cli changefeed create --start-ts=457152641331888132 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ check diff failed 17-th time, retry later go: downloading github.com/jcmturner/aescts/v2 v2.0.0 go: downloading github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b go: downloading github.com/getsentry/sentry-go v0.27.0 go: downloading github.com/go-logr/logr v1.4.1 go: downloading github.com/xdg-go/pbkdf2 v1.0.0 go: downloading github.com/xdg-go/stringprep v1.0.4 go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da go: downloading github.com/golang-jwt/jwt/v4 v4.5.1 go: downloading go.etcd.io/etcd/client/v2 v2.305.12 go: downloading github.com/go-logr/stdr v1.2.2 go: downloading github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1 go: downloading github.com/dvsekhvalnov/jose2go v1.5.0 go: downloading github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c go: downloading github.com/mtibben/percent v0.2.1 go: downloading golang.org/x/term v0.27.0 go: downloading github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.6 go: downloading github.com/aws/aws-sdk-go-v2/internal/ini v1.3.37 go: downloading github.com/aws/aws-sdk-go-v2/service/sso v1.12.14 go: downloading github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.14 go: downloading github.com/aws/aws-sdk-go-v2/service/sts v1.20.1 go: downloading github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.36 go: downloading github.com/kr/pretty v0.3.1 go: downloading github.com/coocood/freecache v1.2.1 go: downloading github.com/pingcap/tipb v0.0.0-20241105053214-f91fdb81a69e go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 go: downloading github.com/pingcap/sysutil v1.0.1-0.20240311050922-ae81ee01f3a5 go: downloading github.com/go-resty/resty/v2 v2.11.0 go: downloading github.com/gorilla/mux v1.8.1 go: downloading github.com/ks3sdklib/aws-sdk-go v1.2.9 go: downloading github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible go: downloading github.com/otiai10/copy v1.2.0 go: downloading github.com/blacktear23/go-proxyprotocol v1.0.6 go: downloading github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.30 go: downloading github.com/pingcap/fn v1.0.0 go: downloading github.com/carlmjohnson/flagext v0.21.0 go: downloading github.com/ardielle/ardielle-go v1.5.2 go: downloading github.com/dolthub/swiss v0.2.1 go: downloading github.com/opentracing/basictracer-go v1.1.0 go: downloading github.com/spkg/bom v1.0.0 go: downloading github.com/xitongsys/parquet-go v1.6.3-0.20240520233950-75e935fc3e17 go: downloading github.com/tidwall/btree v1.7.0 go: downloading github.com/qri-io/jsonschema v0.2.1 Create changefeed successfully! ID: 3bfb11d1-d3f9-498e-8981-cc6ffb4252ab Info: {"upstream_id":7489988830202534805,"namespace":"default","id":"3bfb11d1-d3f9-498e-8981-cc6ffb4252ab","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-06T08:22:15.840529621+08:00","start_ts":457152641331888132,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457152641331888132,"checkpoint_ts":457152641331888132,"checkpoint_time":"2025-04-06 08:22:10.862"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489988859060866752 TiDB Cluster ID. tikv_gc_leader_uuid 65821e7f028000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-2-dwpgc, pid:4104, start at 2025-04-06 08:22:14.320030203 +0800 CST m=+2.368162377 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:24:14.327 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:22:14.332 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:12:14.332 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489988859060866752 TiDB Cluster ID. tikv_gc_leader_uuid 65821e7f028000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-2-dwpgc, pid:4104, start at 2025-04-06 08:22:14.320030203 +0800 CST m=+2.368162377 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:24:14.327 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:22:14.332 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:12:14.332 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489988865561964488 TiDB Cluster ID. tikv_gc_leader_uuid 65821e7f01c0006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-2-dwpgc, pid:4174, start at 2025-04-06 08:22:14.288220748 +0800 CST m=+2.282448781 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:24:14.296 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:22:14.279 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:12:14.279 +0800 All versions after safe point can be accessed. (DO NOT EDIT) go: downloading gopkg.in/yaml.v2 v2.4.0 go: downloading github.com/tiancaiamao/appdash v0.0.0-20181126055449-889f96f722a2 go: downloading github.com/sourcegraph/appdash-data v0.0.0-20151005221446-73f23eafcf67 go: downloading github.com/danjacques/gofslock v0.0.0-20240212154529-d899e02bfe22 go: downloading github.com/dgraph-io/ristretto v0.1.1 go: downloading github.com/cheggaaa/pb/v3 v3.0.8 go: downloading github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 go: downloading go.uber.org/mock v0.4.0 go: downloading github.com/influxdata/tdigest v0.0.1 go: downloading golang.org/x/tools v0.28.0 go: downloading github.com/yangkeao/ldap/v3 v3.4.5-0.20230421065457-369a3bab1117 go: downloading github.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0 go: downloading github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef go: downloading github.com/kr/text v0.2.0 go: downloading github.com/rogpeppe/go-internal v1.13.1 go: downloading github.com/jfcg/sorty/v2 v2.1.0 go: downloading github.com/lestrrat-go/jwx/v2 v2.0.21 go: downloading github.com/jellydator/ttlcache/v3 v3.0.1 go: downloading github.com/google/pprof v0.0.0-20241001023024-f4c0cfd0cf1d go: downloading github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df go: downloading github.com/pingcap/badger v1.5.1-0.20241015064302-38533b6cbf8d go: downloading github.com/fatih/color v1.18.0 go: downloading github.com/vbauerster/mpb/v7 v7.5.3 go: downloading github.com/spf13/cobra v1.8.1 go: downloading github.com/dolthub/maphash v0.1.0 go: downloading github.com/tklauser/go-sysconf v0.3.12 go: downloading github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.30 go: downloading github.com/jedib0t/go-pretty/v6 v6.2.2 table consistent_replicate_gbk.GBKTABLE exists table consistent_replicate_gbk.check1 not exists for 1-th check, retry later [Sun Apr 6 08:22:16 CST 2025] <<<<<< START cdc server in batch_add_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.batch_add_table.54775479.out server --log-file /tmp/tidb_cdc_test/batch_add_table/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/batch_add_table/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/processor_etcd_worker_delay/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/processor_etcd_worker_delay/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/processor_etcd_worker_delay/tiflash/db/proxy"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/processor_etcd_worker_delay/tiflash-proxy.toml"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/processor_etcd_worker_delay/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } go: downloading github.com/robfig/cron/v3 v3.0.1 go: downloading github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f go: downloading github.com/VividCortex/ewma v1.2.0 go: downloading github.com/mattn/go-colorable v0.1.13 go: downloading github.com/mattn/go-runewidth v0.0.16 go: downloading k8s.io/api v0.29.11 go: downloading github.com/emirpasic/gods v1.18.1 go: downloading github.com/jfcg/sixb v1.3.8 go: downloading github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 go: downloading github.com/go-asn1-ber/asn1-ber v1.5.4 go: downloading github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 go: downloading github.com/rivo/uniseg v0.4.7 go: downloading cloud.google.com/go/kms v1.15.8 go: downloading github.com/apache/thrift v0.16.0 go: downloading github.com/apache/arrow/go/v12 v12.0.1 go: downloading github.com/goccy/go-reflect v1.2.0 go: downloading github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d go: downloading github.com/tklauser/numcpus v0.6.1 go: downloading github.com/golang/glog v1.2.0 "id": "1e510708-5e63-4f24-a01f-9ef37aef0e9d", "cluster-id": "default" "cluster-id": "default" run task successfully go: downloading github.com/qri-io/jsonpointer v0.1.1 go: downloading github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c go: downloading github.com/kylelemons/godebug v1.1.0 go: downloading github.com/golang-jwt/jwt/v5 v5.2.1 go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible go: downloading github.com/lestrrat-go/option v1.0.1 go: downloading github.com/lestrrat-go/blackmagic v1.0.2 go: downloading github.com/lestrrat-go/httprc v1.0.5 go: downloading github.com/lestrrat-go/iter v1.0.2 go: downloading github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 go: downloading github.com/klauspost/cpuid v1.3.1 go: downloading github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 go: downloading github.com/ncw/directio v1.0.5 go: downloading github.com/lestrrat-go/httpcc v1.0.1 table ddl_manager.finish_mark not exists for 56-th check, retry later + set +x + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 06 Apr 2025 00:22:17 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8699a8e2-6a28-4206-81d0-0e97468eff15 {"id":"8699a8e2-6a28-4206-81d0-0e97468eff15","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898934} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/2231960879cb57d7 8699a8e2-6a28-4206-81d0-0e97468eff15 /tidb/cdc/default/default/upstream/7489988849654225577 {"id":7489988849654225577,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8699a8e2-6a28-4206-81d0-0e97468eff15 {"id":"8699a8e2-6a28-4206-81d0-0e97468eff15","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898934} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/2231960879cb57d7 8699a8e2-6a28-4206-81d0-0e97468eff15 /tidb/cdc/default/default/upstream/7489988849654225577 {"id":7489988849654225577,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8699a8e2-6a28-4206-81d0-0e97468eff15 {"id":"8699a8e2-6a28-4206-81d0-0e97468eff15","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898934} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/2231960879cb57d7 8699a8e2-6a28-4206-81d0-0e97468eff15 /tidb/cdc/default/default/upstream/7489988849654225577 {"id":7489988849654225577,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x go: downloading golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 go: downloading github.com/google/flatbuffers v2.0.8+incompatible go: downloading github.com/zeebo/xxh3 v1.0.2 go: downloading github.com/andybalholm/brotli v1.0.5 go: downloading github.com/klauspost/cpuid/v2 v2.2.4 go: downloading k8s.io/apimachinery v0.29.11 check diff failed 18-th time, retry later capture_id: 1e510708-5e63-4f24-a01f-9ef37aef0e9d table changefeed_pause_resume.t1 not exists for 1-th check, retry later go: downloading k8s.io/klog/v2 v2.120.1 go: downloading sigs.k8s.io/structured-merge-diff/v4 v4.4.1 go: downloading github.com/google/gofuzz v1.2.0 go: downloading k8s.io/utils v0.0.0-20230726121419-3b25d923346b go: downloading gopkg.in/inf.v0 v0.9.1 go: downloading sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd table consistent_replicate_gbk.check1 exists check diff successfully [Sun Apr 6 08:22:18 CST 2025] <<<<<< START cdc server in processor_etcd_worker_delay case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS='github.com/pingcap/tiflow/pkg/orchestrator/ProcessorEtcdDelay=return(true)' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.processor_etcd_worker_delay.54005402.out server --log-file /tmp/tidb_cdc_test/processor_etcd_worker_delay/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/processor_etcd_worker_delay/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table ddl_manager.finish_mark not exists for 57-th check, retry later wait process cdc.test exit for 1-th time... go: downloading github.com/jmespath/go-jmespath v0.4.0 go: downloading github.com/googleapis/enterprise-certificate-proxy v0.3.2 go: downloading go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 go: downloading github.com/google/s2a-go v0.1.7 go: downloading github.com/felixge/httpsnoop v1.0.4 split_and_random_merge scale: 40 *************************** 1. row *************************** count(distinct region_id): 1 wait process cdc.test exit for 2-th time... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 06 Apr 2025 00:22:19 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/133ae37e-c0c8-4888-bddf-dbb5df047d01 {"id":"133ae37e-c0c8-4888-bddf-dbb5df047d01","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898936} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/2231960879d839db 133ae37e-c0c8-4888-bddf-dbb5df047d01 /tidb/cdc/default/default/upstream/7489988850765574355 {"id":7489988850765574355,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/133ae37e-c0c8-4888-bddf-dbb5df047d01 {"id":"133ae37e-c0c8-4888-bddf-dbb5df047d01","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898936} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/2231960879d839db 133ae37e-c0c8-4888-bddf-dbb5df047d01 /tidb/cdc/default/default/upstream/7489988850765574355 {"id":7489988850765574355,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/133ae37e-c0c8-4888-bddf-dbb5df047d01 {"id":"133ae37e-c0c8-4888-bddf-dbb5df047d01","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898936} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/2231960879d839db 133ae37e-c0c8-4888-bddf-dbb5df047d01 /tidb/cdc/default/default/upstream/7489988850765574355 {"id":7489988850765574355,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.batch_add_table.cli.5540.out cli changefeed create --sink-uri=mysql://normal:123456@127.0.0.1:3306/ check diff failed 19-th time, retry later *************************** 1. row *************************** count(distinct region_id): 40 [2025/04/06 08:22:19.536 +08:00] [INFO] [case.go:544] ["upstream create table executed"] [tblName=finishmark1000] [2025/04/06 08:22:19.536 +08:00] [INFO] [case.go:576] ["downstream start wait for table"] [tblName=finishmark1000] Create changefeed successfully! ID: 39c9408e-c4dc-4553-8825-2530ecb9c5f8 Info: {"upstream_id":7489988850765574355,"namespace":"default","id":"39c9408e-c4dc-4553-8825-2530ecb9c5f8","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-06T08:22:19.638174426+08:00","start_ts":457152643614638081,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457152643614638081,"checkpoint_ts":457152643614638081,"checkpoint_time":"2025-04-06 08:22:19.570"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Sun Apr 6 08:22:19 CST 2025] <<<<<< START cdc server in consistent_replicate_gbk case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sink/dmlsink/txn/mysql/MySQLSinkHangLongTime=return(true)' + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.consistent_replicate_gbk.58795881.out server --log-file /tmp/tidb_cdc_test/consistent_replicate_gbk/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/consistent_replicate_gbk/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table changefeed_pause_resume.t1 exists table changefeed_pause_resume.t2 not exists for 1-th check, retry later table ddl_manager.finish_mark not exists for 58-th check, retry later 0 [Sun Apr 6 08:22:20 CST 2025] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.37853787.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_owner_cleanup_stale_tasks.server3.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_owner_cleanup_stale_tasks.server3 --cluster-id default --addr 127.0.0.1:8302 ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8302; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + set +x =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/batch_update_to_no_batch/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [2025/04/06 08:22:21.536 +08:00] [INFO] [case.go:687] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark1000] [2025/04/06 08:22:21.543 +08:00] [INFO] [case.go:697] ["try parse finishedTs from ticdc log"] [tableName=finishmark1000] /tmp/tidb_cdc_test/bank/cdcdown.log [2025/04/06 08:22:21.544 +08:00] [INFO] [case.go:717] ["total files"] [file="[/tmp/tidb_cdc_test/bank/cdcdown.log]"] table batch_add_table.finish_mark not exists for 1-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 06 Apr 2025 00:22:21 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/98cf9ff2-3273-4ccf-b091-3fc769c1ea81 {"id":"98cf9ff2-3273-4ccf-b091-3fc769c1ea81","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898938} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/2231960879e0ded9 98cf9ff2-3273-4ccf-b091-3fc769c1ea81 /tidb/cdc/default/default/upstream/7489988859060866752 {"id":7489988859060866752,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/98cf9ff2-3273-4ccf-b091-3fc769c1ea81 {"id":"98cf9ff2-3273-4ccf-b091-3fc769c1ea81","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898938} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/2231960879e0ded9 98cf9ff2-3273-4ccf-b091-3fc769c1ea81 /tidb/cdc/default/default/upstream/7489988859060866752 {"id":7489988859060866752,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/98cf9ff2-3273-4ccf-b091-3fc769c1ea81 {"id":"98cf9ff2-3273-4ccf-b091-3fc769c1ea81","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898938} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/2231960879e0ded9 98cf9ff2-3273-4ccf-b091-3fc769c1ea81 /tidb/cdc/default/default/upstream/7489988859060866752 {"id":7489988859060866752,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x check diff failed 20-th time, retry later table changefeed_pause_resume.t2 exists table changefeed_pause_resume.t3 not exists for 1-th check, retry later table ddl_manager.finish_mark not exists for 59-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 06 Apr 2025 00:22:22 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/127d718b-a427-413d-95ca-56806d2114fc {UpstreamID:7489988825213157151 Namespace:default ID:127d718b-a427-413d-95ca-56806d2114fc SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-06 08:22:12.348340139 +0800 CST StartTs:457152641699151876 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003890b40 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457152641712259078} {CheckpointTs:457152644215734275 MinTableBarrierTs:457152644215734276 AdminJobType:noop} span: {tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}, resolvedTs: 457152644215734276, checkpointTs: 457152644215734275, state: Replicating span: {tableID:116, startKey:7480000000000000ff745f720000000000fa, endKey:7480000000000000ff745f730000000000fa}, resolvedTs: 457152644215734275, checkpointTs: 457152644215734275, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/29f98e5a-1d11-47c3-a7e2-6d30ec980e90 {"id":"29f98e5a-1d11-47c3-a7e2-6d30ec980e90","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898940} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/2231960879b92581 29f98e5a-1d11-47c3-a7e2-6d30ec980e90 /tidb/cdc/default/default/changefeed/info/127d718b-a427-413d-95ca-56806d2114fc {"upstream-id":7489988825213157151,"namespace":"default","changefeed-id":"127d718b-a427-413d-95ca-56806d2114fc","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-06T08:22:12.348340139+08:00","start-ts":457152641699151876,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"s3://logbucket/test-changefeed?endpoint=http://127.0.0.1:24927/","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457152641712259078} /tidb/cdc/default/default/changefeed/status/127d718b-a427-413d-95ca-56806d2114fc {"checkpoint-ts":457152644215734275,"min-table-barrier-ts":457152644215734276,"admin-job-type":0} /tidb/cdc/default/default/task/position/29f98e5a-1d11-47c3-a7e2-6d30ec980e90/127d718b-a427-413d-95ca-56806d2114fc {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489988825213157151 {"id":7489988825213157151,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/127d718b-a427-413d-95ca-56806d2114fc {UpstreamID:7489988825213157151 Namespace:default ID:127d718b-a427-413d-95ca-56806d2114fc SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-06 08:22:12.348340139 +0800 CST StartTs:457152641699151876 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003890b40 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457152641712259078} {CheckpointTs:457152644215734275 MinTableBarrierTs:457152644215734276 AdminJobType:noop} span: {tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}, resolvedTs: 457152644215734276, checkpointTs: 457152644215734275, state: Replicating span: {tableID:116, startKey:7480000000000000ff745f720000000000fa, endKey:7480000000000000ff745f730000000000fa}, resolvedTs: 457152644215734275, checkpointTs: 457152644215734275, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/29f98e5a-1d11-47c3-a7e2-6d30ec980e90 {"id":"29f98e5a-1d11-47c3-a7e2-6d30ec980e90","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898940} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/2231960879b92581 29f98e5a-1d11-47c3-a7e2-6d30ec980e90 /tidb/cdc/default/default/changefeed/info/127d718b-a427-413d-95ca-56806d2114fc {"upstream-id":7489988825213157151,"namespace":"default","changefeed-id":"127d718b-a427-413d-95ca-56806d2114fc","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-06T08:22:12.348340139+08:00","start-ts":457152641699151876,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"s3://logbucket/test-changefeed?endpoint=http://127.0.0.1:24927/","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457152641712259078} /tidb/cdc/default/default/changefeed/status/127d718b-a427-413d-95ca-56806d2114fc {"checkpoint-ts":457152644215734275,"min-table-barrier-ts":457152644215734276,"admin-job-type":0} /tidb/cdc/default/default/task/position/29f98e5a-1d11-47c3-a7e2-6d30ec980e90/127d718b-a427-413d-95ca-56806d2114fc {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489988825213157151 + grep -q 'failed to get info:' {"id":7489988825213157151,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/127d718b-a427-413d-95ca-56806d2114fc {UpstreamID:7489988825213157151 Namespace:default ID:127d718b-a427-413d-95ca-56806d2114fc SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-06 08:22:12.348340139 +0800 CST StartTs:457152641699151876 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003890b40 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457152641712259078} {CheckpointTs:457152644215734275 MinTableBarrierTs:457152644215734276 AdminJobType:noop} span: {tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}, resolvedTs: 457152644215734276, checkpointTs: 457152644215734275, state: Replicating span: {tableID:116, startKey:7480000000000000ff745f720000000000fa, endKey:7480000000000000ff745f730000000000fa}, resolvedTs: 457152644215734275, checkpointTs: 457152644215734275, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/29f98e5a-1d11-47c3-a7e2-6d30ec980e90 {"id":"29f98e5a-1d11-47c3-a7e2-6d30ec980e90","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898940} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/2231960879b92581 29f98e5a-1d11-47c3-a7e2-6d30ec980e90 /tidb/cdc/default/default/changefeed/info/127d718b-a427-413d-95ca-56806d2114fc {"upstream-id":7489988825213157151,"namespace":"default","changefeed-id":"127d718b-a427-413d-95ca-56806d2114fc","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-06T08:22:12.348340139+08:00","start-ts":457152641699151876,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"s3://logbucket/test-changefeed?endpoint=http://127.0.0.1:24927/","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457152641712259078} /tidb/cdc/default/default/changefeed/status/127d718b-a427-413d-95ca-56806d2114fc {"checkpoint-ts":457152644215734275,"min-table-barrier-ts":457152644215734276,"admin-job-type":0} /tidb/cdc/default/default/task/position/29f98e5a-1d11-47c3-a7e2-6d30ec980e90/127d718b-a427-413d-95ca-56806d2114fc {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489988825213157151 {"id":7489988825213157151,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x [2025/04/06 08:22:23.446 +08:00] [INFO] [case.go:544] ["upstream create table executed"] [tblName=finishmark2000] [2025/04/06 08:22:23.446 +08:00] [INFO] [case.go:576] ["downstream start wait for table"] [tblName=finishmark2000] table batch_add_table.finish_mark not exists for 2-th check, retry later table ddl_sequence.finish_mark not exists for 1-th check, retry later [2025/04/06 08:22:23.729 +08:00] [INFO] [case.go:687] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark1000] [2025/04/06 08:22:23.738 +08:00] [INFO] [case.go:697] ["try parse finishedTs from ticdc log"] [tableName=finishmark1000] /tmp/tidb_cdc_test/bank/cdcdown.log [2025/04/06 08:22:23.740 +08:00] [INFO] [case.go:717] ["total files"] [file="[/tmp/tidb_cdc_test/bank/cdcdown.log]"] found first match line: 457152644048486414 : [2025/04/06 08:22:22.025 +08:00] [INFO] [ddl_manager.go:298] ["handle a ddl job"] [namespace=default] [changefeed=changefeed-for-find-finished-ts] [tableID=136] [jobID=137] [query="CREATE TABLE IF NOT EXISTS `finishmark1000` (`foo` BIGINT PRIMARY KEY)"] [finishedTs=457152644048486414] [2025/04/06 08:22:23.929 +08:00] [INFO] [case.go:581] ["ddl synced"] [table=finishmark1000] [2025/04/06 08:22:23.929 +08:00] [INFO] [case.go:587] ["downstream sync success"] [endTs=457152644048486414] [2025/04/06 08:22:23.934 +08:00] [INFO] [case.go:191] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2025/04/06 08:22:23.938 +08:00] [INFO] [case.go:191] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2025/04/06 08:22:23.944 +08:00] [INFO] [case.go:308] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] table changefeed_pause_resume.t3 exists table ddl_manager.finish_mark not exists for 60-th check, retry later [2025/04/06 08:22:24.030 +08:00] [INFO] [case.go:308] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8302 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8302 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 06 Apr 2025 00:22:23 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/e684662d-022f-4a79-9ef3-0a59dba96193 {UpstreamID:7489988628380945915 Namespace:default ID:e684662d-022f-4a79-9ef3-0a59dba96193 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-06 08:21:28.032207553 +0800 CST StartTs:457152628763131907 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003b08990 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457152630100066306} {CheckpointTs:457152643023503369 MinTableBarrierTs:457152643023503369 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/02c921f0-7fa5-42dc-8218-4af63b36b5d6 {"id":"02c921f0-7fa5-42dc-8218-4af63b36b5d6","address":"127.0.0.1:8302","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898941} /tidb/cdc/default/__cdc_meta__/capture/1e510708-5e63-4f24-a01f-9ef37aef0e9d {"id":"1e510708-5e63-4f24-a01f-9ef37aef0e9d","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898932} /tidb/cdc/default/__cdc_meta__/capture/b6166d19-012a-453b-adb9-597ea8e2a51a {"id":"b6166d19-012a-453b-adb9-597ea8e2a51a","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898925} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087905471c b6166d19-012a-453b-adb9-597ea8e2a51a /tidb/cdc/default/__cdc_meta__/owner/2231960879054765 1e510708-5e63-4f24-a01f-9ef37aef0e9d /tidb/cdc/default/__cdc_meta__/owner/22319608790547e7 02c921f0-7fa5-42dc-8218-4af63b36b5d6 /tidb/cdc/default/default/changefeed/info/e684662d-022f-4a79-9ef3-0a59dba96193 {"upstream-id":7489988628380945915,"namespace":"default","changefeed-id":"e684662d-022f-4a79-9ef3-0a59dba96193","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-06T08:21:28.032207553+08:00","start-ts":457152628763131907,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457152630100066306} /tidb/cdc/default/default/changefeed/status/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":457152643023503369,"min-table-barrier-ts":457152643023503369,"admin-job-type":0} /tidb/cdc/default/default/task/position/02c921f0-7fa5-42dc-8218-4af63b36b5d6/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/1e510708-5e63-4f24-a01f-9ef37aef0e9d/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/b6166d19-012a-453b-adb9-597ea8e2a51a/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489988628380945915 {"id":7489988628380945915,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/e684662d-022f-4a79-9ef3-0a59dba96193 {UpstreamID:7489988628380945915 Namespace:default ID:e684662d-022f-4a79-9ef3-0a59dba96193 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-06 08:21:28.032207553 +0800 CST StartTs:457152628763131907 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003b08990 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457152630100066306} {CheckpointTs:457152643023503369 MinTableBarrierTs:457152643023503369 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/02c921f0-7fa5-42dc-8218-4af63b36b5d6 {"id":"02c921f0-7fa5-42dc-8218-4af63b36b5d6","address":"127.0.0.1:8302","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898941} /tidb/cdc/default/__cdc_meta__/capture/1e510708-5e63-4f24-a01f-9ef37aef0e9d {"id":"1e510708-5e63-4f24-a01f-9ef37aef0e9d","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898932} /tidb/cdc/default/__cdc_meta__/capture/b6166d19-012a-453b-adb9-597ea8e2a51a {"id":"b6166d19-012a-453b-adb9-597ea8e2a51a","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898925} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087905471c b6166d19-012a-453b-adb9-597ea8e2a51a /tidb/cdc/default/__cdc_meta__/owner/2231960879054765 1e510708-5e63-4f24-a01f-9ef37aef0e9d /tidb/cdc/default/__cdc_meta__/owner/22319608790547e7 02c921f0-7fa5-42dc-8218-4af63b36b5d6 /tidb/cdc/default/default/changefeed/info/e684662d-022f-4a79-9ef3-0a59dba96193 {"upstream-id":7489988628380945915,"namespace":"default","changefeed-id":"e684662d-022f-4a79-9ef3-0a59dba96193","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-06T08:21:28.032207553+08:00","start-ts":457152628763131907,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457152630100066306} + grep -q 'failed to get info:' /tidb/cdc/default/default/changefeed/status/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":457152643023503369,"min-table-barrier-ts":457152643023503369,"admin-job-type":0} /tidb/cdc/default/default/task/position/02c921f0-7fa5-42dc-8218-4af63b36b5d6/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/1e510708-5e63-4f24-a01f-9ef37aef0e9d/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/b6166d19-012a-453b-adb9-597ea8e2a51a/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489988628380945915 {"id":7489988628380945915,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/e684662d-022f-4a79-9ef3-0a59dba96193 {UpstreamID:7489988628380945915 Namespace:default ID:e684662d-022f-4a79-9ef3-0a59dba96193 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-06 08:21:28.032207553 +0800 CST StartTs:457152628763131907 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003b08990 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457152630100066306} {CheckpointTs:457152643023503369 MinTableBarrierTs:457152643023503369 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/02c921f0-7fa5-42dc-8218-4af63b36b5d6 {"id":"02c921f0-7fa5-42dc-8218-4af63b36b5d6","address":"127.0.0.1:8302","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898941} /tidb/cdc/default/__cdc_meta__/capture/1e510708-5e63-4f24-a01f-9ef37aef0e9d {"id":"1e510708-5e63-4f24-a01f-9ef37aef0e9d","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898932} /tidb/cdc/default/__cdc_meta__/capture/b6166d19-012a-453b-adb9-597ea8e2a51a {"id":"b6166d19-012a-453b-adb9-597ea8e2a51a","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898925} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087905471c b6166d19-012a-453b-adb9-597ea8e2a51a /tidb/cdc/default/__cdc_meta__/owner/2231960879054765 1e510708-5e63-4f24-a01f-9ef37aef0e9d /tidb/cdc/default/__cdc_meta__/owner/22319608790547e7 02c921f0-7fa5-42dc-8218-4af63b36b5d6 /tidb/cdc/default/default/changefeed/info/e684662d-022f-4a79-9ef3-0a59dba96193 {"upstream-id":7489988628380945915,"namespace":"default","changefeed-id":"e684662d-022f-4a79-9ef3-0a59dba96193","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-06T08:21:28.032207553+08:00","start-ts":457152628763131907,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457152630100066306} /tidb/cdc/default/default/changefeed/status/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":457152643023503369,"min-table-barrier-ts":457152643023503369,"admin-job-type":0} /tidb/cdc/default/default/task/position/02c921f0-7fa5-42dc-8218-4af63b36b5d6/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/1e510708-5e63-4f24-a01f-9ef37aef0e9d/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/b6166d19-012a-453b-adb9-597ea8e2a51a/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489988628380945915 {"id":7489988628380945915,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x cdc.test cli capture list --server 'http://127.0.0.1:8302' 2>&1 | grep '"is-owner": true' run task failed 1-th time, retry later start tidb cluster in /tmp/tidb_cdc_test/batch_update_to_no_batch Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... check diff successfully wait process cdc.test exit for 1-th time... check diff failed 1-th time, retry later table processor_delay.t1 exists table processor_delay.t2 not exists for 1-th check, retry later wait process cdc.test exit for 2-th time... [2025/04/06 08:22:25.446 +08:00] [INFO] [case.go:687] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark2000] cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Sun Apr 6 08:22:25 CST 2025] <<<<<< run test case clustered_index success! >>>>>> table ddl_sequence.finish_mark not exists for 2-th check, retry later table batch_add_table.finish_mark not exists for 3-th check, retry later cdc.test cli capture list --server 'http://127.0.0.1:8302' 2>&1 | grep '"is-owner": true' split_and_random_merge scale: 80 *************************** 1. row *************************** count(distinct region_id): 1 run task failed 2-th time, retry later table ddl_manager.finish_mark not exists for 61-th check, retry later table processor_delay.t2 not exists for 2-th check, retry later Verifying downstream PD is started... check diff failed 2-th time, retry later Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table batch_add_table.finish_mark not exists for 4-th check, retry later [2025/04/06 08:22:27.525 +08:00] [INFO] [case.go:687] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark2000] [2025/04/06 08:22:27.534 +08:00] [INFO] [case.go:697] ["try parse finishedTs from ticdc log"] [tableName=finishmark2000] /tmp/tidb_cdc_test/bank/cdcdown.log [2025/04/06 08:22:27.536 +08:00] [INFO] [case.go:717] ["total files"] [file="[/tmp/tidb_cdc_test/bank/cdcdown.log]"] found first match line: 457152645175705614 : [2025/04/06 08:22:25.964 +08:00] [INFO] [ddl_manager.go:298] ["handle a ddl job"] [namespace=default] [changefeed=changefeed-for-find-finished-ts] [tableID=138] [jobID=139] [query="CREATE TABLE IF NOT EXISTS `finishmark2000` (`foo` BIGINT PRIMARY KEY)"] [finishedTs=457152645175705614] [2025/04/06 08:22:27.727 +08:00] [INFO] [case.go:581] ["ddl synced"] [table=finishmark2000] [2025/04/06 08:22:27.727 +08:00] [INFO] [case.go:587] ["downstream sync success"] [endTs=457152645175705614] [2025/04/06 08:22:27.730 +08:00] [INFO] [case.go:191] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2025/04/06 08:22:27.733 +08:00] [INFO] [case.go:191] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2025/04/06 08:22:27.737 +08:00] [INFO] [case.go:308] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2025/04/06 08:22:27.743 +08:00] [INFO] [case.go:308] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] table ddl_sequence.finish_mark not exists for 3-th check, retry later table ddl_manager.finish_mark not exists for 62-th check, retry later [2025/04/06 08:22:28.636 +08:00] [INFO] [case.go:544] ["upstream create table executed"] [tblName=finishmark3000] [2025/04/06 08:22:28.636 +08:00] [INFO] [case.go:576] ["downstream start wait for table"] [tblName=finishmark3000] table processor_delay.t2 not exists for 3-th check, retry later valid ~~~ running cdc Failed to start cdc, the usage tips should be printed 1st test case cdc_server_tips success! try an INVALID cdc server command [Sun Apr 6 08:22:27 CST 2025] <<<<<< START cdc server in cdc_server_tips case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ true != \n\o ]] + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cdc_server_tips.54435445.out server --log-file /tmp/tidb_cdc_test/cdc_server_tips/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/cdc_server_tips/cdc_data --cluster-id default --pd None check diff failed 3-th time, retry later ./t/alter_table.test: ok! 64 test cases passed, take time 25.950862265 s time="2025-04-06T08:22:25+08:00" level=info msg="run test [alter_table] ok" time="2025-04-06T08:22:25+08:00" level=warning msg="Create new db&{{{} {} 0} 0xc0000d4438 {{} {} 0} {0 0} [0xc0001bfe00] {[]} 1 0xc00002ce70 false map[0xc0001bfe00:map[0xc0001bfe00:true]] map[] 0 0 0 0 0 0 0 0 0x4a1c00}" check_redo_resolved_ts 127d718b-a427-413d-95ca-56806d2114fc 457152645932777473 s3://logbucket/test-changefeed?endpoint=http://127.0.0.1:24927/ /tmp/tidb_cdc_test/consistent_replicate_gbk/cdc_data/redo/127d718b-a427-413d-95ca-56806d2114fc/meta global resolved ts 457152645788598276 not forward to 457152645932777473 run task failed 1-th time, retry later Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... table batch_add_table.finish_mark not exists for 5-th check, retry later Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_sequence.finish_mark not exists for 4-th check, retry later table ddl_manager.finish_mark not exists for 63-th check, retry later cdc.test cli capture list --server 'http://127.0.0.1:8302' 2>&1 | grep '"is-owner": true' check_redo_resolved_ts 127d718b-a427-413d-95ca-56806d2114fc 457152645932777473 s3://logbucket/test-changefeed?endpoint=http://127.0.0.1:24927/ /tmp/tidb_cdc_test/consistent_replicate_gbk/cdc_data/redo/127d718b-a427-413d-95ca-56806d2114fc/meta run task successfully wait process cdc.test exit for 1-th time... [2025/04/06 08:22:30.637 +08:00] [INFO] [case.go:687] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark3000] [2025/04/06 08:22:30.643 +08:00] [INFO] [case.go:697] ["try parse finishedTs from ticdc log"] [tableName=finishmark3000] /tmp/tidb_cdc_test/bank/cdcdown.log [2025/04/06 08:22:30.645 +08:00] [INFO] [case.go:717] ["total files"] [file="[/tmp/tidb_cdc_test/bank/cdcdown.log]"] check diff failed 4-th time, retry later table processor_delay.t2 exists table processor_delay.t3 not exists for 1-th check, retry later table batch_add_table.finish_mark not exists for 6-th check, retry later wait process cdc.test exit for 2-th time... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) "is-owner": true, run task successfully nonempty select id, val from test.availability1 where id=1 and val=1 table ddl_sequence.finish_mark exists check diff successfully TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [2025/04/06 08:22:32.242 +08:00] [INFO] [helper.go:54] ["init log"] [file=] [level=info] [2025/04/06 08:22:32.243 +08:00] [INFO] [apply.go:78] ["set memory limit"] [memoryLimit=10737418240] [2025/04/06 08:22:32.243 +08:00] [INFO] [apply.go:94] ["Start http pprof server"] [addr=127.0.0.1:6060] [2025/04/06 08:22:32.244 +08:00] [INFO] [s3.go:423] ["succeed to get bucket region from s3"] ["bucket region"=us-east-1] [2025/04/06 08:22:32.247 +08:00] [INFO] [mem_quota.go:82] ["New memory quota"] [namespace=default] [changefeed=redo-applier] [total=1073741824] [2025/04/06 08:22:32.247 +08:00] [INFO] [redo.go:167] ["apply redo log starts"] [checkpointTs=457152644451663894] [resolvedTs=457152646312886276] [2025/04/06 08:22:32.248 +08:00] [INFO] [s3.go:423] ["succeed to get bucket region from s3"] ["bucket region"=us-east-1] [2025/04/06 08:22:32.248 +08:00] [INFO] [tz.go:35] ["Use the timezone of the TiCDC server machine"] [timezoneName=System] [timezone=Asia/Shanghai] [2025/04/06 08:22:32.248 +08:00] [WARN] [config.go:428] ["Because time-zone is not specified, the timezone of the TiCDC server will be used. We recommend that you specify the time-zone explicitly. Please make sure that the timezone of the TiCDC server, sink-uri and the downstream database are consistent. If the downstream database does not load the timezone information, you can refer to https://dev.mysql.com/doc/refman/8.0/en/mysql-tzinfo-to-sql.html."] [timezone=Asia/Shanghai] [2025/04/06 08:22:32.248 +08:00] [ERROR] [config.go:216] ["The TiDB source ID should never be set to 0. Please report it as a bug. The default value will be used: 1."] [tidbSourceID=0] [2025/04/06 08:22:32.248 +08:00] [INFO] [s3.go:423] ["succeed to get bucket region from s3"] ["bucket region"=us-east-1] [2025/04/06 08:22:32.251 +08:00] [INFO] [file.go:114] ["succeed to download and sort redo logs"] [type=ddl] [duration=4.022709ms] [2025/04/06 08:22:32.251 +08:00] [INFO] [file.go:114] ["succeed to download and sort redo logs"] [type=row] [duration=4.084088ms] [2025/04/06 08:22:32.253 +08:00] [INFO] [db_helper.go:161] ["sink uri is configured"] [dsn="normal:******@tcp(127.0.0.1:3306)/?interpolateParams=true&multiStatements=true&allow_auto_random_explicit_insert=1&charset=utf8mb4&foreign_key_checks=0&maxAllowedPacket=0&readTimeout=2m&sql_mode=%22NO_AUTO_VALUE_ON_ZERO%2CONLY_FULL_GROUP_BY%2CNO_ENGINE_SUBSTITUTION%2CALLOW_INVALID_DATES%2CIGNORE_SPACE%22&tidb_enable_external_ts_read=%22OFF%22&tidb_placement_mode=%22ignore%22&tidb_txn_mode=optimistic&time_zone=%22Asia%2FShanghai%22&timeout=2m&transaction_isolation=%22READ-COMMITTED%22&writeTimeout=2m"] [2025/04/06 08:22:32.255 +08:00] [INFO] [mysql.go:195] ["MySQL backends is created"] [changefeed=.] [workerCount=16] [forceReplicate=false] [2025/04/06 08:22:32.257 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/06 08:22:32.257 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/06 08:22:32.257 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/06 08:22:32.257 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/06 08:22:32.257 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/06 08:22:32.257 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/06 08:22:32.257 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/06 08:22:32.257 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/06 08:22:32.257 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/06 08:22:32.257 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/06 08:22:32.257 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/06 08:22:32.257 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/06 08:22:32.257 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/06 08:22:32.257 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/06 08:22:32.257 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/06 08:22:32.257 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/06 08:22:32.257 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=1] [2025/04/06 08:22:32.257 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=3] [2025/04/06 08:22:32.257 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=4] [2025/04/06 08:22:32.257 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=0] [2025/04/06 08:22:32.257 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=2] [2025/04/06 08:22:32.257 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=7] [2025/04/06 08:22:32.257 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=9] [2025/04/06 08:22:32.257 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=5] [2025/04/06 08:22:32.257 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=11] [2025/04/06 08:22:32.257 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=12] [2025/04/06 08:22:32.257 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=10] [2025/04/06 08:22:32.257 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=8] [2025/04/06 08:22:32.257 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=6] [2025/04/06 08:22:32.257 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=13] [2025/04/06 08:22:32.257 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=15] [2025/04/06 08:22:32.257 +08:00] [INFO] [tz.go:35] ["Use the timezone of the TiCDC server machine"] [timezoneName=System] [timezone=Asia/Shanghai] [2025/04/06 08:22:32.257 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=14] [2025/04/06 08:22:32.257 +08:00] [WARN] [config.go:428] ["Because time-zone is not specified, the timezone of the TiCDC server will be used. We recommend that you specify the time-zone explicitly. Please make sure that the timezone of the TiCDC server, sink-uri and the downstream database are consistent. If the downstream database does not load the timezone information, you can refer to https://dev.mysql.com/doc/refman/8.0/en/mysql-tzinfo-to-sql.html."] [timezone=Asia/Shanghai] [2025/04/06 08:22:32.257 +08:00] [ERROR] [config.go:216] ["The TiDB source ID should never be set to 0. Please report it as a bug. The default value will be used: 1."] [tidbSourceID=0] [2025/04/06 08:22:32.261 +08:00] [INFO] [db_helper.go:161] ["sink uri is configured"] [dsn="normal:******@tcp(127.0.0.1:3306)/?interpolateParams=true&multiStatements=true&allow_auto_random_explicit_insert=1&charset=utf8mb4&foreign_key_checks=0&maxAllowedPacket=0&readTimeout=2m&sql_mode=%22IGNORE_SPACE%2CALLOW_INVALID_DATES%2CONLY_FULL_GROUP_BY%2CNO_AUTO_VALUE_ON_ZERO%2CNO_ENGINE_SUBSTITUTION%22&tidb_enable_external_ts_read=%22OFF%22&tidb_placement_mode=%22ignore%22&tidb_txn_mode=optimistic&time_zone=%22Asia%2FShanghai%22&timeout=2m&transaction_isolation=%22READ-COMMITTED%22&writeTimeout=2m"] [2025/04/06 08:22:32.263 +08:00] [WARN] [mysql_ddl_sink.go:291] ["please set `has-vector-type` to be true if a column is vector type when the downstream is not TiDB or TiDB version less than specify version"] [hasVectorType=false] [supportVectorVersion=8.4.0] [2025/04/06 08:22:32.263 +08:00] [INFO] [mysql_ddl_sink.go:122] ["MySQL DDL sink is created"] [namespace=] [changefeed=] [2025/04/06 08:22:32.263 +08:00] [WARN] [redo.go:287] ["apply DDL"] [ddl="{\"StartTs\":457152644451663890,\"CommitTs\":457152644451663894,\"Query\":\"create table consistent_replicate_gbk.GBKTABLE2 like consistent_replicate_gbk.GBKTABLE\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"Schema\":\"consistent_replicate_gbk\",\"Table\":\"GBKTABLE2\",\"TableID\":118,\"IsPartition\":false},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":3,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false,\"BDRRole\":\"\",\"SQLMode\":0}"] [2025/04/06 08:22:32.265 +08:00] [INFO] [mysql_ddl_sink.go:225] ["Start exec DDL"] [namespace=] [changefeed=] [commitTs=457152644451663894] [DDL="create table consistent_replicate_gbk.GBKTABLE2 like consistent_replicate_gbk.GBKTABLE"] [2025/04/06 08:22:32.267 +08:00] [INFO] [mysql_ddl_sink.go:150] ["Execute DDL failed, but error can be ignored"] [startTs=457152644451663890] [ddl="create table consistent_replicate_gbk.GBKTABLE2 like consistent_replicate_gbk.GBKTABLE"] [namespace=] [changefeed=] [error="Error 1050 (42S01): Table 'consistent_replicate_gbk.GBKTABLE2' already exists"] table ddl_manager.finish_mark not exists for 64-th check, retry later [2025/04/06 08:22:32.467 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [span="{tableID:118, startKey:7480000000000000ff765f720000000000fa, endKey:7480000000000000ff765f730000000000fa}"] [checkpointTs=457152646312886276] [2025/04/06 08:22:32.467 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [span="{tableID:118, startKey:7480000000000000ff765f720000000000fa, endKey:7480000000000000ff765f730000000000fa}"] [checkpointTs=457152646312886276] [2025/04/06 08:22:32.568 +08:00] [INFO] [redo.go:223] ["apply redo log finishes"] [appliedLogCount=1] [appliedDDLCount=1] [currentCheckpoint=457152646312886276] [2025/04/06 08:22:32.568 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=3] [2025/04/06 08:22:32.568 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=10] [2025/04/06 08:22:32.568 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=14] [2025/04/06 08:22:32.568 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=15] [2025/04/06 08:22:32.568 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=1] [2025/04/06 08:22:32.568 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=4] [2025/04/06 08:22:32.568 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=5] [2025/04/06 08:22:32.568 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=11] [2025/04/06 08:22:32.568 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=8] [2025/04/06 08:22:32.568 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=6] [2025/04/06 08:22:32.568 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=2] [2025/04/06 08:22:32.568 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=13] [2025/04/06 08:22:32.568 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=7] [2025/04/06 08:22:32.568 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=9] [2025/04/06 08:22:32.568 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=12] [2025/04/06 08:22:32.568 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=0] Apply redo log successfully wait process cdc.test exit for 1-th time... *************************** 1. row *************************** count(distinct region_id): 80 check diff successfully [Sun Apr 6 08:22:32 CST 2025] <<<<<< run test case consistent_replicate_gbk success! >>>>>> Exiting on signal: INTERRUPT ***************** properties ***************** "dotransactions"="false" "recordcount"="10" "mysql.host"="127.0.0.1" "workload"="core" "requestdistribution"="uniform" "updateproportion"="0" "scanproportion"="0" "mysql.db"="testSync" "readproportion"="0" "mysql.port"="4000" "insertproportion"="0" "readallfields"="true" "mysql.user"="root" "threadcount"="2" "operationcount"="0" ********************************************** Run finished, takes 6.382154ms INSERT - Takes(s): 0.0, Count: 10, OPS: 2456.8, Avg(us): 1108, Min(us): 491, Max(us): 3708, 95th(us): 4000, 99th(us): 4000 [2025/04/06 08:22:32.929 +08:00] [INFO] [case.go:687] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark3000] [2025/04/06 08:22:32.935 +08:00] [INFO] [case.go:697] ["try parse finishedTs from ticdc log"] [tableName=finishmark3000] /tmp/tidb_cdc_test/bank/cdcdown.log [2025/04/06 08:22:32.938 +08:00] [INFO] [case.go:717] ["total files"] [file="[/tmp/tidb_cdc_test/bank/cdcdown.log]"] wait process cdc.test exit for 2-th time... table processor_delay.t3 exists table processor_delay.t4 not exists for 1-th check, retry later found first match line: 457152646407782413 : [2025/04/06 08:22:31.125 +08:00] [INFO] [ddl_manager.go:298] ["handle a ddl job"] [namespace=default] [changefeed=changefeed-for-find-finished-ts] [tableID=140] [jobID=141] [query="CREATE TABLE IF NOT EXISTS `finishmark3000` (`foo` BIGINT PRIMARY KEY)"] [finishedTs=457152646407782413] [2025/04/06 08:22:33.140 +08:00] [INFO] [case.go:581] ["ddl synced"] [table=finishmark3000] [2025/04/06 08:22:33.140 +08:00] [INFO] [case.go:587] ["downstream sync success"] [endTs=457152646407782413] [2025/04/06 08:22:33.142 +08:00] [INFO] [case.go:191] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2025/04/06 08:22:33.146 +08:00] [INFO] [case.go:191] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2025/04/06 08:22:33.150 +08:00] [INFO] [case.go:308] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2025/04/06 08:22:33.157 +08:00] [INFO] [case.go:308] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] table batch_add_table.finish_mark not exists for 7-th check, retry later cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Sun Apr 6 08:22:33 CST 2025] <<<<<< run test case ddl_sequence success! >>>>>> check diff failed 5-th time, retry later nonempty select id, val from test.availability1 where id=1 and val=1 run task successfully nonempty select id, val from test.availability1 where id=1 and val=22 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later table ddl_manager.finish_mark not exists for 65-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489988937398098457 TiDB Cluster ID. tikv_gc_leader_uuid 65821e90b200006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-4-x87nv, pid:3924, start at 2025-04-06 08:22:32.439238538 +0800 CST m=+2.600776506 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:24:32.447 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:22:32.443 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:12:32.443 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489988937398098457 TiDB Cluster ID. tikv_gc_leader_uuid 65821e90b200006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-4-x87nv, pid:3924, start at 2025-04-06 08:22:32.439238538 +0800 CST m=+2.600776506 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:24:32.447 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:22:32.443 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:12:32.443 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489988940909470161 TiDB Cluster ID. tikv_gc_leader_uuid 65821e90cb40006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-4-x87nv, pid:3986, start at 2025-04-06 08:22:32.502538972 +0800 CST m=+2.606695819 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:24:32.544 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:22:32.543 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:12:32.543 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/batch_update_to_no_batch/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/batch_update_to_no_batch/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/batch_update_to_no_batch/tiflash/log/proxy.log"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/batch_update_to_no_batch/tiflash-proxy.toml"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/batch_update_to_no_batch/tiflash/db/proxy"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [2025/04/06 08:22:34.327 +08:00] [INFO] [case.go:544] ["upstream create table executed"] [tblName=finishmark4000] [2025/04/06 08:22:34.327 +08:00] [INFO] [case.go:576] ["downstream start wait for table"] [tblName=finishmark4000] table processor_delay.t4 exists table processor_delay.t5 not exists for 1-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/processor_resolved_ts_fallback/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... check diff successfully table batch_add_table.finish_mark not exists for 8-th check, retry later [2025/04/06 08:22:36.327 +08:00] [INFO] [case.go:687] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark4000] [2025/04/06 08:22:36.336 +08:00] [INFO] [case.go:697] ["try parse finishedTs from ticdc log"] [tableName=finishmark4000] /tmp/tidb_cdc_test/bank/cdcdown.log [2025/04/06 08:22:36.337 +08:00] [INFO] [case.go:717] ["total files"] [file="[/tmp/tidb_cdc_test/bank/cdcdown.log]"] check diff failed 1-th time, retry later nonempty select id, val from test.availability1 where id=1 and val=22 run task successfully empty select id, val from test.availability1 where id=1 TEST FAILED: OUTPUT CONTAINS 'id:' ____________________________________ *************************** 1. row *************************** id: 1 val: 22 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later table ddl_manager.finish_mark not exists for 66-th check, retry later found first match line: 457152647583236113 : [2025/04/06 08:22:35.064 +08:00] [INFO] [ddl_manager.go:298] ["handle a ddl job"] [namespace=default] [changefeed=changefeed-for-find-finished-ts] [tableID=142] [jobID=143] [query="CREATE TABLE IF NOT EXISTS `finishmark4000` (`foo` BIGINT PRIMARY KEY)"] [finishedTs=457152647583236113] [2025/04/06 08:22:36.731 +08:00] [INFO] [case.go:581] ["ddl synced"] [table=finishmark4000] [2025/04/06 08:22:36.731 +08:00] [INFO] [case.go:587] ["downstream sync success"] [endTs=457152647583236113] [2025/04/06 08:22:36.734 +08:00] [INFO] [case.go:191] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2025/04/06 08:22:36.825 +08:00] [INFO] [case.go:191] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2025/04/06 08:22:36.833 +08:00] [INFO] [case.go:308] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2025/04/06 08:22:36.840 +08:00] [INFO] [case.go:308] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] table processor_delay.t5 exists table processor_delay.t6 not exists for 1-th check, retry later table batch_add_table.finish_mark not exists for 9-th check, retry later table region_merge.t1 exists check diff failed 1-th time, retry later start tidb cluster in /tmp/tidb_cdc_test/processor_resolved_ts_fallback Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... empty select id, val from test.availability1 where id=1 run task successfully test_owner_cleanup_stale_tasks pass [Sun Apr 6 08:22:38 CST 2025] <<<<<< START cdc server in batch_update_to_no_batch case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.batch_update_to_no_batch.52435245.out server --log-file /tmp/tidb_cdc_test/batch_update_to_no_batch/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/batch_update_to_no_batch/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table ddl_manager.finish_mark not exists for 67-th check, retry later wait process cdc.test exit for 1-th time... table processor_delay.t6 exists table processor_delay.t7 not exists for 1-th check, retry later check diff failed 2-th time, retry later wait process cdc.test exit for 2-th time... [2025/04/06 08:22:38.935 +08:00] [INFO] [case.go:544] ["upstream create table executed"] [tblName=finishmark5000] [2025/04/06 08:22:38.935 +08:00] [INFO] [case.go:576] ["downstream start wait for table"] [tblName=finishmark5000] Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit run test case test_owner_retryable_error [Sun Apr 6 08:22:39 CST 2025] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/capture/capture-campaign-compacted-error=1*return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.40444046.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_owner_retryable_error.server1.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_owner_retryable_error.server1 --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table batch_add_table.finish_mark not exists for 10-th check, retry later check diff successfully wait process cdc.test exit for 1-th time... [2025/04/06 08:22:37.690 +08:00] [WARN] [diff.go:180] ["table struct is not equal"] [reason="column num not equal, one is 34 another is 15"] table ddl_manager.finish_mark not exists for 68-th check, retry later wait process cdc.test exit for 2-th time... check diff failed 3-th time, retry later [2025/04/06 08:22:40.935 +08:00] [INFO] [case.go:687] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark5000] [2025/04/06 08:22:41.032 +08:00] [INFO] [case.go:697] ["try parse finishedTs from ticdc log"] [tableName=finishmark5000] /tmp/tidb_cdc_test/bank/cdcdown.log [2025/04/06 08:22:41.033 +08:00] [INFO] [case.go:717] ["total files"] [file="[/tmp/tidb_cdc_test/bank/cdcdown.log]"] cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Sun Apr 6 08:22:41 CST 2025] <<<<<< run test case region_merge success! >>>>>> table processor_delay.t7 exists table processor_delay.t8 not exists for 1-th check, retry later found first match line: 457152649029222414 : [2025/04/06 08:22:41.126 +08:00] [INFO] [ddl_manager.go:298] ["handle a ddl job"] [namespace=default] [changefeed=changefeed-for-find-finished-ts] [tableID=144] [jobID=145] [query="CREATE TABLE IF NOT EXISTS `finishmark5000` (`foo` BIGINT PRIMARY KEY)"] [finishedTs=457152649029222414] [2025/04/06 08:22:41.431 +08:00] [INFO] [case.go:581] ["ddl synced"] [table=finishmark5000] [2025/04/06 08:22:41.431 +08:00] [INFO] [case.go:587] ["downstream sync success"] [endTs=457152649029222414] [2025/04/06 08:22:41.434 +08:00] [INFO] [case.go:191] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2025/04/06 08:22:41.439 +08:00] [INFO] [case.go:191] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2025/04/06 08:22:41.533 +08:00] [INFO] [case.go:308] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2025/04/06 08:22:41.625 +08:00] [INFO] [case.go:308] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] table batch_add_table.finish_mark not exists for 11-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 06 Apr 2025 00:22:41 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d14f57b6-e06a-4f92-a047-ab02c86a9f3c {"id":"d14f57b6-e06a-4f92-a047-ab02c86a9f3c","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898958} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087a22b9ec d14f57b6-e06a-4f92-a047-ab02c86a9f3c /tidb/cdc/default/default/upstream/7489988937398098457 {"id":7489988937398098457,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d14f57b6-e06a-4f92-a047-ab02c86a9f3c {"id":"d14f57b6-e06a-4f92-a047-ab02c86a9f3c","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898958} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087a22b9ec d14f57b6-e06a-4f92-a047-ab02c86a9f3c /tidb/cdc/default/default/upstream/7489988937398098457 {"id":7489988937398098457,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d14f57b6-e06a-4f92-a047-ab02c86a9f3c {"id":"d14f57b6-e06a-4f92-a047-ab02c86a9f3c","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898958} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087a22b9ec d14f57b6-e06a-4f92-a047-ab02c86a9f3c /tidb/cdc/default/default/upstream/7489988937398098457 {"id":7489988937398098457,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.batch_update_to_no_batch.cli.5295.out cli changefeed create '--sink-uri=mysql://root@127.0.0.1:3306/?batch-dml-enable=true' -c test Create changefeed successfully! ID: test Info: {"upstream_id":7489988937398098457,"namespace":"default","id":"test","sink_uri":"mysql://root@127.0.0.1:3306/?batch-dml-enable=true","create_time":"2025-04-06T08:22:41.85449585+08:00","start_ts":457152649426894851,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457152649426894851,"checkpoint_ts":457152649426894851,"checkpoint_time":"2025-04-06 08:22:41.742"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 69-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/consistent_replicate_nfs/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... check diff failed 4-th time, retry later [2025/04/06 08:22:42.694 +08:00] [WARN] [diff.go:180] ["table struct is not equal"] [reason="column num not equal, one is 34 another is 28"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 06 Apr 2025 00:22:42 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/e684662d-022f-4a79-9ef3-0a59dba96193 {UpstreamID:7489988628380945915 Namespace:default ID:e684662d-022f-4a79-9ef3-0a59dba96193 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-06 08:21:28.032207553 +0800 CST StartTs:457152628763131907 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00489a5a0 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457152630100066306} {CheckpointTs:457152648279752707 MinTableBarrierTs:457152649590210564 AdminJobType:noop} span: {tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}, resolvedTs: 457152648279752707, checkpointTs: 457152648279752707, state: Preparing span: {tableID:112, startKey:7480000000000000ff705f720000000000fa, endKey:7480000000000000ff705f730000000000fa}, resolvedTs: 457152648279752707, checkpointTs: 457152648279752707, state: Preparing span: {tableID:116, startKey:7480000000000000ff745f720000000000fa, endKey:7480000000000000ff745f730000000000fa}, resolvedTs: 457152648279752707, checkpointTs: 457152648279752707, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f2518902-4f98-4d6e-9adb-7fe81522c41e {"id":"f2518902-4f98-4d6e-9adb-7fe81522c41e","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898960} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 1 /tidb/cdc/default/__cdc_meta__/owner/22319608790548a5 f2518902-4f98-4d6e-9adb-7fe81522c41e /tidb/cdc/default/default/changefeed/info/e684662d-022f-4a79-9ef3-0a59dba96193 {"upstream-id":7489988628380945915,"namespace":"default","changefeed-id":"e684662d-022f-4a79-9ef3-0a59dba96193","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-06T08:21:28.032207553+08:00","start-ts":457152628763131907,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457152630100066306} /tidb/cdc/default/default/changefeed/status/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":457152648279752707,"min-table-barrier-ts":457152649590210564,"admin-job-type":0} /tidb/cdc/default/default/task/position/f2518902-4f98-4d6e-9adb-7fe81522c41e/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489988628380945915 {"id":7489988628380945915,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/e684662d-022f-4a79-9ef3-0a59dba96193 {UpstreamID:7489988628380945915 Namespace:default ID:e684662d-022f-4a79-9ef3-0a59dba96193 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-06 08:21:28.032207553 +0800 CST StartTs:457152628763131907 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00489a5a0 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457152630100066306} {CheckpointTs:457152648279752707 MinTableBarrierTs:457152649590210564 AdminJobType:noop} span: {tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}, resolvedTs: 457152648279752707, checkpointTs: 457152648279752707, state: Preparing span: {tableID:112, startKey:7480000000000000ff705f720000000000fa, endKey:7480000000000000ff705f730000000000fa}, resolvedTs: 457152648279752707, checkpointTs: 457152648279752707, state: Preparing span: {tableID:116, startKey:7480000000000000ff745f720000000000fa, endKey:7480000000000000ff745f730000000000fa}, resolvedTs: 457152648279752707, checkpointTs: 457152648279752707, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f2518902-4f98-4d6e-9adb-7fe81522c41e {"id":"f2518902-4f98-4d6e-9adb-7fe81522c41e","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898960} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 1 /tidb/cdc/default/__cdc_meta__/owner/22319608790548a5 f2518902-4f98-4d6e-9adb-7fe81522c41e /tidb/cdc/default/default/changefeed/info/e684662d-022f-4a79-9ef3-0a59dba96193 {"upstream-id":7489988628380945915,"namespace":"default","changefeed-id":"e684662d-022f-4a79-9ef3-0a59dba96193","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-06T08:21:28.032207553+08:00","start-ts":457152628763131907,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457152630100066306} /tidb/cdc/default/default/changefeed/status/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":457152648279752707,"min-table-barrier-ts":457152649590210564,"admin-job-type":0} /tidb/cdc/default/default/task/position/f2518902-4f98-4d6e-9adb-7fe81522c41e/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489988628380945915 {"id":7489988628380945915,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/e684662d-022f-4a79-9ef3-0a59dba96193 {UpstreamID:7489988628380945915 Namespace:default ID:e684662d-022f-4a79-9ef3-0a59dba96193 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-06 08:21:28.032207553 +0800 CST StartTs:457152628763131907 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00489a5a0 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457152630100066306} {CheckpointTs:457152648279752707 MinTableBarrierTs:457152649590210564 AdminJobType:noop} span: {tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}, resolvedTs: 457152648279752707, checkpointTs: 457152648279752707, state: Preparing span: {tableID:112, startKey:7480000000000000ff705f720000000000fa, endKey:7480000000000000ff705f730000000000fa}, resolvedTs: 457152648279752707, checkpointTs: 457152648279752707, state: Preparing span: {tableID:116, startKey:7480000000000000ff745f720000000000fa, endKey:7480000000000000ff745f730000000000fa}, resolvedTs: 457152648279752707, checkpointTs: 457152648279752707, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f2518902-4f98-4d6e-9adb-7fe81522c41e {"id":"f2518902-4f98-4d6e-9adb-7fe81522c41e","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898960} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 1 /tidb/cdc/default/__cdc_meta__/owner/22319608790548a5 f2518902-4f98-4d6e-9adb-7fe81522c41e /tidb/cdc/default/default/changefeed/info/e684662d-022f-4a79-9ef3-0a59dba96193 {"upstream-id":7489988628380945915,"namespace":"default","changefeed-id":"e684662d-022f-4a79-9ef3-0a59dba96193","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-06T08:21:28.032207553+08:00","start-ts":457152628763131907,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457152630100066306} /tidb/cdc/default/default/changefeed/status/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":457152648279752707,"min-table-barrier-ts":457152649590210564,"admin-job-type":0} /tidb/cdc/default/default/task/position/f2518902-4f98-4d6e-9adb-7fe81522c41e/e684662d-022f-4a79-9ef3-0a59dba96193 + grep -q 'etcd info' {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489988628380945915 {"id":7489988628380945915,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list 2>&1 | grep '"is-owner": true' table processor_delay.t8 exists table processor_delay.t9 not exists for 1-th check, retry later + set +x table batch_add_table.finish_mark not exists for 12-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ./t/bigint.test: ok! 138 test cases passed, take time 15.340648467 s time="2025-04-06T08:22:41+08:00" level=info msg="run test [bigint] ok" time="2025-04-06T08:22:41+08:00" level=warning msg="Create new db&{{{} {} 0} 0xc0001ac788 {{} {} 0} {0 0} [0xc000420d80] {[]} 1 0xc0001b1e30 false map[0xc000420d80:map[0xc000420d80:true]] map[] 0 0 0 0 0 0 0 0 0x4a1c00}" ./t/composite_index.test: ok! 12 test cases passed, take time 1.600286315 s time="2025-04-06T08:22:43+08:00" level=info msg="run test [composite_index] ok" time="2025-04-06T08:22:43+08:00" level=warning msg="Create new db&{{{} {} 0} 0xc000310048 {{} {} 0} {0 0} [0xc0002ea000] {[]} 1 0xc00002c5b0 false map[0xc0002ea000:map[0xc0002ea000:true]] map[] 0 0 0 0 0 0 0 0 0x4a1c00}" [2025/04/06 08:22:43.737 +08:00] [INFO] [case.go:544] ["upstream create table executed"] [tblName=finishmark6000] [2025/04/06 08:22:43.737 +08:00] [INFO] [case.go:576] ["downstream start wait for table"] [tblName=finishmark6000] table ddl_manager.finish_mark not exists for 70-th check, retry later "is-owner": true, run task successfully table processor_delay.t9 exists table processor_delay.t10 not exists for 1-th check, retry later check diff failed 5-th time, retry later ./t/concurrent_ddl.test: ok! 8 test cases passed, take time 1.022372063 s time="2025-04-06T08:22:45+08:00" level=info msg="run test [concurrent_ddl] ok" time="2025-04-06T08:22:45+08:00" level=warning msg="Create new db&{{{} {} 0} 0xc0001ac078 {{} {} 0} {0 0} [0xc000490100] {[]} 1 0xc0001b0230 false map[0xc000490100:map[0xc000490100:true]] map[] 0 0 0 0 0 0 0 0 0x4a1c00}" owner pid: 4049 owner id f2518902-4f98-4d6e-9adb-7fe81522c41e [Sun Apr 6 08:22:45 CST 2025] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/owner-run-with-error=1*return(true);github.com/pingcap/tiflow/cdc/capture/capture-resign-failed=1*return(true)' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.41644166.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_owner_retryable_error.server2.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_owner_retryable_error.server2 --cluster-id default --addr 127.0.0.1:8301 ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/resourcecontrol/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/consistent_replicate_nfs Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489988993064913331 TiDB Cluster ID. tikv_gc_leader_uuid 65821e9bfc40009 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-j-x7t22, pid:4311, start at 2025-04-06 08:22:43.967059988 +0800 CST m=+2.133191860 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:24:43.974 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:22:43.953 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:12:43.953 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489988993064913331 TiDB Cluster ID. tikv_gc_leader_uuid 65821e9bfc40009 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-j-x7t22, pid:4311, start at 2025-04-06 08:22:43.967059988 +0800 CST m=+2.133191860 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:24:43.974 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:22:43.953 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:12:43.953 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489988995912362382 TiDB Cluster ID. tikv_gc_leader_uuid 65821e9c4180005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-j-x7t22, pid:4374, start at 2025-04-06 08:22:44.236358799 +0800 CST m=+2.332991538 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:24:44.258 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:22:44.230 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:12:44.230 +0800 All versions after safe point can be accessed. (DO NOT EDIT) [2025/04/06 08:22:45.737 +08:00] [INFO] [case.go:687] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark6000] [2025/04/06 08:22:45.746 +08:00] [INFO] [case.go:697] ["try parse finishedTs from ticdc log"] [tableName=finishmark6000] /tmp/tidb_cdc_test/bank/cdcdown.log [2025/04/06 08:22:45.747 +08:00] [INFO] [case.go:717] ["total files"] [file="[/tmp/tidb_cdc_test/bank/cdcdown.log]"] table batch_add_table.finish_mark not exists for 13-th check, retry later Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/processor_resolved_ts_fallback/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/processor_resolved_ts_fallback/tiflash/log/error.log arg matches is ArgMatches { args: {"config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/processor_resolved_ts_fallback/tiflash-proxy.toml"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/processor_resolved_ts_fallback/tiflash/db/proxy"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/processor_resolved_ts_fallback/tiflash/log/proxy.log"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } found first match line: 457152650339942413 : [2025/04/06 08:22:46.225 +08:00] [INFO] [ddl_manager.go:298] ["handle a ddl job"] [namespace=default] [changefeed=changefeed-for-find-finished-ts] [tableID=146] [jobID=147] [query="CREATE TABLE IF NOT EXISTS `finishmark6000` (`foo` BIGINT PRIMARY KEY)"] [finishedTs=457152650339942413] [2025/04/06 08:22:46.336 +08:00] [INFO] [case.go:581] ["ddl synced"] [table=finishmark6000] [2025/04/06 08:22:46.336 +08:00] [INFO] [case.go:587] ["downstream sync success"] [endTs=457152650339942413] [2025/04/06 08:22:46.338 +08:00] [INFO] [case.go:191] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2025/04/06 08:22:46.341 +08:00] [INFO] [case.go:191] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2025/04/06 08:22:46.426 +08:00] [INFO] [case.go:308] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2025/04/06 08:22:46.435 +08:00] [INFO] [case.go:308] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] table ddl_manager.finish_mark not exists for 71-th check, retry later table processor_delay.t10 exists table processor_delay.t11 not exists for 1-th check, retry later check diff successfully invalid ~~~ running cdc Failed to start cdc, the usage tips should be printed 2nd test case cdc_server_tips success! [Sun Apr 6 08:22:47 CST 2025] <<<<<< run all test cases cdc_server_tips success! >>>>>> table batch_add_table.finish_mark not exists for 14-th check, retry later check diff failed 1-th time, retry later [Sun Apr 6 08:22:48 CST 2025] <<<<<< START cdc server in processor_resolved_ts_fallback case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sink/dmlsink/txn/mysql/MySQLSinkExecDMLError=return(true);github.com/pingcap/tiflow/cdc/sink/dmlsink/mq/dmlproducer/KafkaSinkAsyncSendError=return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.processor_resolved_ts_fallback.56105612.out server --log-file /tmp/tidb_cdc_test/processor_resolved_ts_fallback/cdc1.log --log-level debug --data-dir /tmp/tidb_cdc_test/processor_resolved_ts_fallback/cdc_data1 --cluster-id default --addr 127.0.0.1:8301 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.batch_update_to_no_batch.cli.5363.out cli changefeed pause -c test table ddl_manager.finish_mark not exists for 72-th check, retry later PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 06 Apr 2025 00:22:48 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/e684662d-022f-4a79-9ef3-0a59dba96193 {UpstreamID:7489988628380945915 Namespace:default ID:e684662d-022f-4a79-9ef3-0a59dba96193 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-06 08:21:28.032207553 +0800 CST StartTs:457152628763131907 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00432e1b0 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457152630100066306} {CheckpointTs:457152651163074566 MinTableBarrierTs:457152651163074566 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c63ffb24-748c-45dd-afca-b589945a7a1f {"id":"c63ffb24-748c-45dd-afca-b589945a7a1f","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898966} /tidb/cdc/default/__cdc_meta__/capture/f2518902-4f98-4d6e-9adb-7fe81522c41e {"id":"f2518902-4f98-4d6e-9adb-7fe81522c41e","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898960} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 1 /tidb/cdc/default/__cdc_meta__/owner/22319608790548a5 f2518902-4f98-4d6e-9adb-7fe81522c41e /tidb/cdc/default/__cdc_meta__/owner/22319608790548ec c63ffb24-748c-45dd-afca-b589945a7a1f /tidb/cdc/default/default/changefeed/info/e684662d-022f-4a79-9ef3-0a59dba96193 {"upstream-id":7489988628380945915,"namespace":"default","changefeed-id":"e684662d-022f-4a79-9ef3-0a59dba96193","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-06T08:21:28.032207553+08:00","start-ts":457152628763131907,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457152630100066306} /tidb/cdc/default/default/changefeed/status/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":457152651163074566,"min-table-barrier-ts":457152651163074566,"admin-job-type":0} /tidb/cdc/default/default/task/position/c63ffb24-748c-45dd-afca-b589945a7a1f/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/f2518902-4f98-4d6e-9adb-7fe81522c41e/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489988628380945915 {"id":7489988628380945915,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/e684662d-022f-4a79-9ef3-0a59dba96193 {UpstreamID:7489988628380945915 Namespace:default ID:e684662d-022f-4a79-9ef3-0a59dba96193 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-06 08:21:28.032207553 +0800 CST StartTs:457152628763131907 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00432e1b0 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457152630100066306} {CheckpointTs:457152651163074566 MinTableBarrierTs:457152651163074566 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c63ffb24-748c-45dd-afca-b589945a7a1f {"id":"c63ffb24-748c-45dd-afca-b589945a7a1f","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898966} /tidb/cdc/default/__cdc_meta__/capture/f2518902-4f98-4d6e-9adb-7fe81522c41e {"id":"f2518902-4f98-4d6e-9adb-7fe81522c41e","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898960} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 1 /tidb/cdc/default/__cdc_meta__/owner/22319608790548a5 f2518902-4f98-4d6e-9adb-7fe81522c41e /tidb/cdc/default/__cdc_meta__/owner/22319608790548ec c63ffb24-748c-45dd-afca-b589945a7a1f /tidb/cdc/default/default/changefeed/info/e684662d-022f-4a79-9ef3-0a59dba96193 {"upstream-id":7489988628380945915,"namespace":"default","changefeed-id":"e684662d-022f-4a79-9ef3-0a59dba96193","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-06T08:21:28.032207553+08:00","start-ts":457152628763131907,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457152630100066306} /tidb/cdc/default/default/changefeed/status/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":457152651163074566,"min-table-barrier-ts":457152651163074566,"admin-job-type":0} /tidb/cdc/default/default/task/position/c63ffb24-748c-45dd-afca+ grep -q 'failed to get info:' -b589945a7a1f/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/f2518902-4f98-4d6e-9adb-7fe81522c41e/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489988628380945915 {"id":7489988628380945915,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/e684662d-022f-4a79-9ef3-0a59dba96193 {UpstreamID:7489988628380945915 Namespace:default ID:e684662d-022f-4a79-9ef3-0a59dba96193 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-06 08:21:28.032207553 +0800 CST StartTs:457152628763131907 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00432e1b0 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457152630100066306} {CheckpointTs:457152651163074566 MinTableBarrierTs:457152651163074566 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c63ffb24-748c-45dd-afca-b589945a7a1f {"id":"c63ffb24-748c-45dd-afca-b589945a7a1f","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898966} /tidb/cdc/default/__cdc_meta__/capture/f2518902-4f98-4d6e-9adb-7fe81522c41e {"id":"f2518902-4f98-4d6e-9adb-7fe81522c41e","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898960} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 1 /tidb/cdc/default/__cdc_meta__/owner/22319608790548a5 f2518902-4f98-4d6e-9adb-7fe81522c41e /tidb/cdc/default/__cdc_meta__/owner/22319608790548ec c63ffb24-748c-45dd-afca-b589945a7a1f /tidb/cdc/default/default/changefeed/info/e684662d-022f-4a79-9ef3-0a59dba96193 {"upstream-id":7489988628380945915,"namespace":"default","changefeed-id":"e684662d-022f-4a79-9ef3-0a59dba96193","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-06T08:21:28.032207553+08:00","start-ts":457152628763131907,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457152630100066306} /tidb/cdc/default/default/changefeed/status/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":457152651163074566,"min-table-barrier-ts":457152651163074566,"admin-job-type":0} /tidb/cdc/default/default/task/position/c63ffb24-748c-45dd-afca-b589945a7a1f/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/f2518902-4f98-4d6e-9adb-7fe81522c41e/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489988628380945915 {"id":7489988628380945915,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x cdc.test cli capture list 2>&1 | grep -v "f2518902-4f98-4d6e-9adb-7fe81522c41e" | grep id [2025/04/06 08:22:48.827 +08:00] [INFO] [case.go:544] ["upstream create table executed"] [tblName=finishmark7000] [2025/04/06 08:22:48.827 +08:00] [INFO] [case.go:576] ["downstream start wait for table"] [tblName=finishmark7000] start tidb cluster in /tmp/tidb_cdc_test/resourcecontrol Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... table processor_delay.t11 exists table processor_delay.t12 not exists for 1-th check, retry later table batch_add_table.finish_mark not exists for 15-th check, retry later check diff failed 2-th time, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.batch_update_to_no_batch.cli.5389.out cli changefeed update -c test '--sink-uri=mysql://root@127.0.0.1:3306/?batch-dml-enable=false' --no-confirm "cluster-id": "default" "id": "c63ffb24-748c-45dd-afca-b589945a7a1f", "cluster-id": "default" run task successfully table ddl_manager.finish_mark not exists for 73-th check, retry later Diff of changefeed config: {Type:update Path:[SinkURI] From:mysql://root@127.0.0.1:3306/?batch-dml-enable=true To:mysql://root@127.0.0.1:3306/?batch-dml-enable=false} [2025/04/06 08:22:50.828 +08:00] [INFO] [case.go:687] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark7000] [2025/04/06 08:22:50.837 +08:00] [INFO] [case.go:697] ["try parse finishedTs from ticdc log"] [tableName=finishmark7000] /tmp/tidb_cdc_test/bank/cdcdown.log [2025/04/06 08:22:50.839 +08:00] [INFO] [case.go:717] ["total files"] [file="[/tmp/tidb_cdc_test/bank/cdcdown.log]"] Update changefeed config successfully! ID: test Info: {"upstream_id":7489988937398098457,"namespace":"default","id":"test","sink_uri":"mysql://root@127.0.0.1:3306/?batch-dml-enable=false","create_time":"2025-04-06T08:22:41.85449585+08:00","start_ts":457152649426894851,"admin_job_type":1,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"delete_only_output_handle_key_columns":null,"content_compatible":null,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"stopped","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":0,"checkpoint_ts":457152650239541269,"checkpoint_time":"2025-04-06 08:22:44.842"} PASS coverage: 2.1% of statements in github.com/pingcap/tiflow/... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/common_1/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... table processor_delay.t12 exists table processor_delay.t13 not exists for 1-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 06 Apr 2025 00:22:51 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ec14bd97-b429-4398-99c3-1ead494fd5f9 {"id":"ec14bd97-b429-4398-99c3-1ead494fd5f9","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898968} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087a5996d6 ec14bd97-b429-4398-99c3-1ead494fd5f9 /tidb/cdc/default/default/upstream/7489988993064913331 {"id":7489988993064913331,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ec14bd97-b429-4398-99c3-1ead494fd5f9 {"id":"ec14bd97-b429-4398-99c3-1ead494fd5f9","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898968} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087a5996d6 ec14bd97-b429-4398-99c3-1ead494fd5f9 /tidb/cdc/default/default/upstream/7489988993064913331 {"id":7489988993064913331,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ec14bd97-b429-4398-99c3-1ead494fd5f9 {"id":"ec14bd97-b429-4398-99c3-1ead494fd5f9","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898968} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087a5996d6 ec14bd97-b429-4398-99c3-1ead494fd5f9 /tidb/cdc/default/default/upstream/7489988993064913331 {"id":7489988993064913331,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.processor_resolved_ts_fallback.cli.5668.out cli changefeed create --sink-uri=mysql://normal:123456@127.0.0.1:3306/ --server=127.0.0.1:8301 Starting Upstream TiDB... found first match line: 457152651646468109 : [2025/04/06 08:22:51.225 +08:00] [INFO] [ddl_manager.go:298] ["handle a ddl job"] [namespace=default] [changefeed=changefeed-for-find-finished-ts] [tableID=148] [jobID=149] [query="CREATE TABLE IF NOT EXISTS `finishmark7000` (`foo` BIGINT PRIMARY KEY)"] [finishedTs=457152651646468109] [2025/04/06 08:22:51.434 +08:00] [INFO] [case.go:581] ["ddl synced"] [table=finishmark7000] [2025/04/06 08:22:51.434 +08:00] [INFO] [case.go:587] ["downstream sync success"] [endTs=457152651646468109] [2025/04/06 08:22:51.525 +08:00] [INFO] [case.go:191] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2025/04/06 08:22:51.530 +08:00] [INFO] [case.go:191] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2025/04/06 08:22:51.537 +08:00] [INFO] [case.go:308] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] Create changefeed successfully! ID: b28becbb-2b1b-490a-917e-fa5c05af6460 Info: {"upstream_id":7489988993064913331,"namespace":"default","id":"b28becbb-2b1b-490a-917e-fa5c05af6460","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-06T08:22:51.642014259+08:00","start_ts":457152652007440386,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457152652007440386,"checkpoint_ts":457152652007440386,"checkpoint_time":"2025-04-06 08:22:51.586"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table batch_add_table.finish_mark not exists for 16-th check, retry later capture_id: c63ffb24-748c-45dd-afca-b589945a7a1f % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 35 100 35 0 0 33175 0 --:--:-- --:--:-- --:--:-- 35000 { "status": true, "message": "" }ETCDCTL_API=3 etcdctl get /tidb/cdc/default/__cdc_meta__/owner --prefix | grep 'f2518902-4f98-4d6e-9adb-7fe81522c41e' f2518902-4f98-4d6e-9adb-7fe81522c41e run task successfully ps -C cdc.test -o pid= | awk '{print $1}' | wc -l | grep 2 2 run task successfully test_owner_retryable_error pass [2025/04/06 08:22:51.630 +08:00] [INFO] [case.go:308] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] ./t/date_formats.test: ok! 99 test cases passed, take time 5.961406711 s time="2025-04-06T08:22:51+08:00" level=info msg="run test [date_formats] ok" time="2025-04-06T08:22:51+08:00" level=warning msg="Create new db&{{{} {} 0} 0xc0004803f0 {{} {} 0} {0 0} [0xc0001f4580] {[]} 1 0xc00049b030 false map[0xc0001f4580:map[0xc0001f4580:true]] map[] 0 0 0 0 0 0 0 0 0x4a1c00}" wait process cdc.test exit for 1-th time... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table ddl_manager.finish_mark not exists for 74-th check, retry later + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.batch_update_to_no_batch.cli.5433.out cli changefeed resume -c test check diff failed 3-th time, retry later wait process cdc.test exit for 2-th time... PASS coverage: 2.1% of statements in github.com/pingcap/tiflow/... + set +x table processor_delay.t13 exists table processor_delay.t14 not exists for 1-th check, retry later cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit run test case test_gap_between_watch_capture [Sun Apr 6 08:22:53 CST 2025] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/sleep-in-owner-tick=1*sleep(6000)' + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.43114313.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_gap_between_watch_capture.server1.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_gap_between_watch_capture.server1 --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table batch_add_table.finish_mark not exists for 17-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... [2025/04/06 08:22:53.737 +08:00] [INFO] [case.go:544] ["upstream create table executed"] [tblName=finishmark8000] [2025/04/06 08:22:53.737 +08:00] [INFO] [case.go:576] ["downstream start wait for table"] [tblName=finishmark8000] table ddl_manager.finish_mark not exists for 75-th check, retry later + set +x check diff failed 4-th time, retry later cdc cli processor list --server http://127.0.0.1:8301 |jq '.|length'|grep -E '^1$' 1 run task successfully table processor_resolved_ts_fallback.t1 exists [Sun Apr 6 08:22:54 CST 2025] <<<<<< START cdc server in processor_resolved_ts_fallback case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.processor_resolved_ts_fallback.57445746.out server --log-file /tmp/tidb_cdc_test/processor_resolved_ts_fallback/cdc2.log --log-level debug --data-dir /tmp/tidb_cdc_test/processor_resolved_ts_fallback/cdc_data2 --cluster-id default --addr 127.0.0.1:8302 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8302; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 start tidb cluster in /tmp/tidb_cdc_test/common_1 Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table processor_delay.t14 exists table processor_delay.t15 not exists for 1-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489989039230729915 TiDB Cluster ID. tikv_gc_leader_uuid 65821ea5a200003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-6-zhp59, pid:7105, start at 2025-04-06 08:22:53.833024501 +0800 CST m=+2.295724321 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:24:53.842 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:22:53.832 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:12:53.832 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489989039230729915 TiDB Cluster ID. tikv_gc_leader_uuid 65821ea5a200003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-6-zhp59, pid:7105, start at 2025-04-06 08:22:53.833024501 +0800 CST m=+2.295724321 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:24:53.842 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:22:53.832 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:12:53.832 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489989030161289617 TiDB Cluster ID. tikv_gc_leader_uuid 65821ea5bb40004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-6-zhp59, pid:7169, start at 2025-04-06 08:22:53.938609853 +0800 CST m=+2.301585149 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:24:53.945 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:22:53.933 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:12:53.933 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/consistent_replicate_nfs/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/consistent_replicate_nfs/tiflash/log/error.log arg matches is ArgMatches { args: {"memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/consistent_replicate_nfs/tiflash/db/proxy"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/consistent_replicate_nfs/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/consistent_replicate_nfs/tiflash-proxy.toml"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table batch_add_table.finish_mark exists check diff successfully [2025/04/06 08:22:55.737 +08:00] [INFO] [case.go:687] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark8000] [2025/04/06 08:22:55.828 +08:00] [INFO] [case.go:697] ["try parse finishedTs from ticdc log"] [tableName=finishmark8000] /tmp/tidb_cdc_test/bank/cdcdown.log [2025/04/06 08:22:55.830 +08:00] [INFO] [case.go:717] ["total files"] [file="[/tmp/tidb_cdc_test/bank/cdcdown.log]"] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.batch_update_to_no_batch.cli.5468.out cli changefeed pause -c test check diff successfully Verifying downstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 1-th time... PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 06 Apr 2025 00:22:56 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/e684662d-022f-4a79-9ef3-0a59dba96193 {UpstreamID:7489988628380945915 Namespace:default ID:e684662d-022f-4a79-9ef3-0a59dba96193 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-06 08:21:28.032207553 +0800 CST StartTs:457152628763131907 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0042ce510 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457152630100066306} {CheckpointTs:457152651949506567 MinTableBarrierTs:457152651949506567 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/11473620-8337-41a3-80cc-aea294cfac6f {"id":"11473620-8337-41a3-80cc-aea294cfac6f","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898973} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 1 /tidb/cdc/default/__cdc_meta__/owner/22319608790548ec c63ffb24-748c-45dd-afca-b589945a7a1f /tidb/cdc/default/__cdc_meta__/owner/223196087905496e 11473620-8337-41a3-80cc-aea294cfac6f /tidb/cdc/default/default/changefeed/info/e684662d-022f-4a79-9ef3-0a59dba96193 {"upstream-id":7489988628380945915,"namespace":"default","changefeed-id":"e684662d-022f-4a79-9ef3-0a59dba96193","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-06T08:21:28.032207553+08:00","start-ts":457152628763131907,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457152630100066306} /tidb/cdc/default/default/changefeed/status/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":457152651949506567,"min-table-barrier-ts":457152651949506567,"admin-job-type":0} /tidb/cdc/default/default/task/position/11473620-8337-41a3-80cc-aea294cfac6f/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489988628380945915 {"id":7489988628380945915,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/e684662d-022f-4a79-9ef3-0a59dba96193 {UpstreamID:7489988628380945915 Namespace:default ID:e684662d-022f-4a79-9ef3-0a59dba96193 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-06 08:21:28.032207553 +0800 CST StartTs:457152628763131907 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0042ce510 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457152630100066306} {CheckpointTs:457152651949506567 MinTableBarrierTs:457152651949506567 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/11473620-8337-41a3-80cc-aea294cfac6f {"id":"11473620-8337-41a3-80cc-aea294cfac6f","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898973} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 1 /tidb/cdc/default/__cdc_meta__/owner/22319608790548ec c63ffb24-748c-45dd-afca-b589945a7a1f /tidb/cdc/default/__cdc_meta__/owner/223196087905496e 11473620-8337-41a3-80cc-aea294cfac6f /tidb/cdc/default/default/changefeed/info/e684662d-022f-4a79-9ef3-0a59dba96193 {"upstream-id":7489988628380945915,"namespace":"default","changefeed-id":"e684662d-022f-4a79-9ef3-0a59dba96193","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-06T08:21:28.032207553+08:00","start-ts":457152628763131907,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457152630100066306} /tidb/cdc/default/default/changefeed/status/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":457152651949506567,"min-table-barrier-ts":457152651949506567,"admin-job-type":0} /tidb/cdc/default/default/task/position/11473620-8337-41a3-80cc-aea294cfac6f/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489988628380945915 {"id":7489988628380945915,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: changefeedID: default/e684662d-022f-4a79-9ef3-0a59dba96193 {UpstreamID:7489988628380945915 Namespace:default ID:e684662d-022f-4a79-9ef3-0a59dba96193 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-06 08:21:28.032207553 +0800 CST StartTs:457152628763131907 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0042ce510 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457152630100066306} {CheckpointTs:457152651949506567 MinTableBarrierTs:457152651949506567 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/11473620-8337-41a3-80cc-aea294cfac6f {"id":"11473620-8337-41a3-80cc-aea294cfac6f","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898973} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 1 /tidb/cdc/default/__cdc_meta__/owner/22319608790548ec c63ffb24-748c-45dd-afca-b589945a7a1f /tidb/cdc/default/__cdc_meta__/owner/223196087905496e 11473620-8337-41a3-80cc-aea294cfac6f /tidb/cdc/default/default/changefeed/info/e684662d-022f-4a79-9ef3-0a59dba96193 {"upstream-id":7489988628380945915,"namespace":"default","changefeed-id":"e684662d-022f-4a79-9ef3-0a59dba96193","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-06T08:21:28.032207553+08:00","start-ts":457152628763131907,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457152630100066306} /tidb/cdc/default/default/changefeed/status/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":457152651949506567,"min-table-barrier-ts":457152651949506567,"admin-job-type":0} /tidb/cdc/default/default/task/position/11473620-8337-41a3-80cc-aea294cfac6f/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489988628380945915 {"id":7489988628380945915,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x cdc.test cli capture list 2>&1 | grep '"is-owner": true' run task failed 1-th time, retry later found first match line: 457152652987596814 : [2025/04/06 08:22:56.325 +08:00] [INFO] [ddl_manager.go:298] ["handle a ddl job"] [namespace=default] [changefeed=changefeed-for-find-finished-ts] [tableID=150] [jobID=151] [query="CREATE TABLE IF NOT EXISTS `finishmark8000` (`foo` BIGINT PRIMARY KEY)"] [finishedTs=457152652987596814] [2025/04/06 08:22:56.527 +08:00] [INFO] [case.go:581] ["ddl synced"] [table=finishmark8000] [2025/04/06 08:22:56.527 +08:00] [INFO] [case.go:587] ["downstream sync success"] [endTs=457152652987596814] [2025/04/06 08:22:56.530 +08:00] [INFO] [case.go:191] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2025/04/06 08:22:56.535 +08:00] [INFO] [case.go:191] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2025/04/06 08:22:56.629 +08:00] [INFO] [case.go:308] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2025/04/06 08:22:56.636 +08:00] [INFO] [case.go:308] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] table ddl_manager.finish_mark not exists for 76-th check, retry later wait process cdc.test exit for 2-th time... table processor_delay.t15 exists table processor_delay.t16 not exists for 1-th check, retry later cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit check diff failed 1-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8302 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8302 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 06 Apr 2025 00:22:57 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/b28becbb-2b1b-490a-917e-fa5c05af6460 {UpstreamID:7489988993064913331 Namespace:default ID:b28becbb-2b1b-490a-917e-fa5c05af6460 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-06 08:22:51.642014259 +0800 CST StartTs:457152652007440386 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003a94f30 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457152652020547586} {CheckpointTs:457152653396541448 MinTableBarrierTs:457152653396541448 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d8e4ea99-6f2c-43f5-af53-ca028d73b8b3 {"id":"d8e4ea99-6f2c-43f5-af53-ca028d73b8b3","address":"127.0.0.1:8302","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898974} /tidb/cdc/default/__cdc_meta__/capture/ec14bd97-b429-4398-99c3-1ead494fd5f9 {"id":"ec14bd97-b429-4398-99c3-1ead494fd5f9","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898968} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087a5996d6 ec14bd97-b429-4398-99c3-1ead494fd5f9 /tidb/cdc/default/__cdc_meta__/owner/223196087a59974f d8e4ea99-6f2c-43f5-af53-ca028d73b8b3 /tidb/cdc/default/default/changefeed/info/b28becbb-2b1b-490a-917e-fa5c05af6460 {"upstream-id":7489988993064913331,"namespace":"default","changefeed-id":"b28becbb-2b1b-490a-917e-fa5c05af6460","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-06T08:22:51.642014259+08:00","start-ts":457152652007440386,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457152652020547586} /tidb/cdc/default/default/changefeed/status/b28becbb-2b1b-490a-917e-fa5c05af6460 {"checkpoint-ts":457152653396541448,"min-table-barrier-ts":457152653396541448,"admin-job-type":0} /tidb/cdc/default/default/task/position/d8e4ea99-6f2c-43f5-af53-ca028d73b8b3/b28becbb-2b1b-490a-917e-fa5c05af6460 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/ec14bd97-b429-4398-99c3-1ead494fd5f9/b28becbb-2b1b-490a-917e-fa5c05af6460 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489988993064913331 {"id":7489988993064913331,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/b28becbb-2b1b-490a-917e-fa5c05af6460 {UpstreamID:7489988993064913331 Namespace:default ID:b28becbb-2b1b-490a-917e-fa5c05af6460 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-06 08:22:51.642014259 +0800 CST StartTs:457152652007440386 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003a94f30 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457152652020547586} {CheckpointTs:457152653396541448 MinTableBarrierTs:457152653396541448 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d8e4ea99-6f2c-43f5-af53-ca028d73b8b3 {"id":"d8e4ea99-6f2c-43f5-af53-ca028d73b8b3","address":"127.0.0.1:8302","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898974} /tidb/cdc/default/__cdc_meta__/capture/ec14bd97-b429-4398-99c3-1ead494fd5f9 {"id":"ec14bd97-b429-4398-99c3-1ead494fd5f9","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898968} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087a5996d6 ec14bd97-b429-4398-99c3-1ead494fd5f9 /tidb/cdc/default/__cdc_meta__/owner/223196087a59974f d8e4ea99-6f2c-43f5-af53-ca028d73b8b3 /tidb/cdc/default/default/changefeed/info/b28becbb-2b1b-490a-917e-fa5c05af6460 {"upstream-id":7489988993064913331,"namespace":"default","changefeed-id":"b28becbb-2b1b-490a-917e-fa5c05af6460","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-06T08:22:51.642014259+08:00","start-ts":457152652007440386,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457152652020547586} /tidb/cdc/default/default/changefeed/status/b28becbb-2b1b-490a-917e-fa5c05af6460 {"checkpoint-ts":457152653396541448,"min-table-barrier-ts":457152653396541448,"admin-job-type":0} /tidb/cdc/default/default/task/position/d8e4ea99-6f2c-43f5-af53-ca028d73b8b3/b28becbb-2b1b-490a-917e-fa5c05af6460 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/ec14bd97-b429-4398-99c3-1ead494fd5f9/b28becbb-2b1b-490a-917e-fa5c05af6460 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489988993064913331 {"id":7489988993064913331,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: changefeedID: default/b28becbb-2b1b-490a-917e-fa5c05af6460 {UpstreamID:7489988993064913331 Namespace:default ID:b28becbb-2b1b-490a-917e-fa5c05af6460 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-06 08:22:51.642014259 +0800 CST StartTs:457152652007440386 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003a94f30 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457152652020547586} {CheckpointTs:457152653396541448 MinTableBarrierTs:457152653396541448 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d8e4ea99-6f2c-43f5-af53-ca028d73b8b3 {"id":"d8e4ea99-6f2c-43f5-af53-ca028d73b8b3","address":"127.0.0.1:8302","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898974} /tidb/cdc/default/__cdc_meta__/capture/ec14bd97-b429-4398-99c3-1ead494fd5f9 {"id":"ec14bd97-b429-4398-99c3-1ead494fd5f9","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898968} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087a5996d6 ec14bd97-b429-4398-99c3-1ead494fd5f9 /tidb/cdc/default/__cdc_meta__/owner/223196087a59974f d8e4ea99-6f2c-43f5-af53-ca028d73b8b3 /tidb/cdc/default/default/changefeed/info/b28becbb-2b1b-490a-917e-fa5c05af6460 {"upstream-id":7489988993064913331,"namespace":"default","changefeed-id":"b28becbb-2b1b-490a-917e-fa5c05af6460","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-06T08:22:51.642014259+08:00","start-ts":457152652007440386,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457152652020547586} /tidb/cdc/default/default/changefeed/status/b28becbb-2b1b-490a-917e-fa5c05af6460 {"checkpoint-ts":457152653396541448,"min-table-barrier-ts":457152653396541448,"admin-job-type":0} /tidb/cdc/default/default/task/position/d8e4ea99-6f2c-43f5-af53-ca028d73b8b3/b28becbb-2b1b-490a-917e-fa5c05af6460 {"checkpoint-ts"+ grep -q 'etcd info' :0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/ec14bd97-b429-4398-99c3-1ead494fd5f9/b28becbb-2b1b-490a-917e-fa5c05af6460 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489988993064913331 {"id":7489988993064913331,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.batch_update_to_no_batch.cli.5493.out cli changefeed update -c test '--sink-uri=mysql://root@127.0.0.1:3306/?batch-dml-enable=true' --no-confirm [Sun Apr 6 08:22:57 CST 2025] <<<<<< START cdc server in consistent_replicate_nfs case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.consistent_replicate_nfs.85008502.out server --log-file /tmp/tidb_cdc_test/consistent_replicate_nfs/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/consistent_replicate_nfs/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Diff of changefeed config: {Type:update Path:[SinkURI] From:mysql://root@127.0.0.1:3306/?batch-dml-enable=false To:mysql://root@127.0.0.1:3306/?batch-dml-enable=true} Update changefeed config successfully! ID: test Info: {"upstream_id":7489988937398098457,"namespace":"default","id":"test","sink_uri":"mysql://root@127.0.0.1:3306/?batch-dml-enable=true","create_time":"2025-04-06T08:22:41.85449585+08:00","start_ts":457152649426894851,"admin_job_type":1,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"delete_only_output_handle_key_columns":null,"content_compatible":null,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"stopped","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":0,"checkpoint_ts":457152650383720450,"checkpoint_time":"2025-04-06 08:22:45.392"} PASS coverage: 2.1% of statements in github.com/pingcap/tiflow/... [2025/04/06 08:22:58.528 +08:00] [INFO] [case.go:544] ["upstream create table executed"] [tblName=finishmark9000] [2025/04/06 08:22:58.528 +08:00] [INFO] [case.go:576] ["downstream start wait for table"] [tblName=finishmark9000] table processor_resolved_ts_fallback.t2 not exists for 1-th check, retry later table ddl_manager.finish_mark not exists for 77-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489989052176982054 TiDB Cluster ID. tikv_gc_leader_uuid 65821ea89740004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-m-cm2pq, pid:10368, start at 2025-04-06 08:22:56.866556212 +0800 CST m=+2.554209669 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:24:56.875 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:22:56.861 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:12:56.861 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489989052176982054 TiDB Cluster ID. tikv_gc_leader_uuid 65821ea89740004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-m-cm2pq, pid:10368, start at 2025-04-06 08:22:56.866556212 +0800 CST m=+2.554209669 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:24:56.875 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:22:56.861 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:12:56.861 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489989041782178288 TiDB Cluster ID. tikv_gc_leader_uuid 65821ea8a880009 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-m-cm2pq, pid:10436, start at 2025-04-06 08:22:56.948254137 +0800 CST m=+2.578652572 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:24:56.957 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:22:56.958 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:12:56.958 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/resourcecontrol/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/resourcecontrol/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/resourcecontrol/tiflash/db/proxy"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/resourcecontrol/tiflash-proxy.toml"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/resourcecontrol/tiflash/log/proxy.log"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } cdc.test cli capture list 2>&1 | grep '"is-owner": true' run task failed 2-th time, retry later table processor_delay.t16 exists table processor_delay.t17 not exists for 1-th check, retry later check diff failed 2-th time, retry later + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.batch_update_to_no_batch.cli.5533.out cli changefeed resume -c test Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) PASS [2025/04/06 08:23:00.529 +08:00] [INFO] [case.go:687] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark9000] table processor_resolved_ts_fallback.t2 exists table processor_resolved_ts_fallback.t3 not exists for 1-th check, retry later coverage: 2.1% of statements in github.com/pingcap/tiflow/... table ddl_manager.finish_mark not exists for 78-th check, retry later + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.resourcecontrol.cli.11673.out cli tso query --pd=http://127.0.0.1:2379 [2025/04/06 08:23:00.538 +08:00] [INFO] [case.go:697] ["try parse finishedTs from ticdc log"] [tableName=finishmark9000] /tmp/tidb_cdc_test/bank/cdcdown.log [2025/04/06 08:23:00.539 +08:00] [INFO] [case.go:717] ["total files"] [file="[/tmp/tidb_cdc_test/bank/cdcdown.log]"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 06 Apr 2025 00:23:01 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/44999fb6-d301-4358-8eef-30d9e7976be9 {"id":"44999fb6-d301-4358-8eef-30d9e7976be9","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898978} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087a77a5d6 44999fb6-d301-4358-8eef-30d9e7976be9 /tidb/cdc/default/default/upstream/7489989039230729915 {"id":7489989039230729915,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/44999fb6-d301-4358-8eef-30d9e7976be9 {"id":"44999fb6-d301-4358-8eef-30d9e7976be9","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898978} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087a77a5d6 44999fb6-d301-4358-8eef-30d9e7976be9 /tidb/cdc/default/default/upstream/7489989039230729915 {"id":7489989039230729915,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/44999fb6-d301-4358-8eef-30d9e7976be9 {"id":"44999fb6-d301-4358-8eef-30d9e7976be9","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898978} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087a77a5d6 44999fb6-d301-4358-8eef-30d9e7976be9 /tidb/cdc/default/default/upstream/7489989039230729915 {"id":7489989039230729915,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x found first match line: 457152654036172815 : [2025/04/06 08:23:00.325 +08:00] [INFO] [ddl_manager.go:298] ["handle a ddl job"] [namespace=default] [changefeed=changefeed-for-find-finished-ts] [tableID=152] [jobID=153] [query="CREATE TABLE IF NOT EXISTS `finishmark9000` (`foo` BIGINT PRIMARY KEY)"] [finishedTs=457152654036172815] [2025/04/06 08:23:01.133 +08:00] [INFO] [case.go:581] ["ddl synced"] [table=finishmark9000] [2025/04/06 08:23:01.133 +08:00] [INFO] [case.go:587] ["downstream sync success"] [endTs=457152654036172815] [2025/04/06 08:23:01.135 +08:00] [INFO] [case.go:191] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2025/04/06 08:23:01.140 +08:00] [INFO] [case.go:191] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2025/04/06 08:23:01.147 +08:00] [INFO] [case.go:308] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2025/04/06 08:23:01.153 +08:00] [INFO] [case.go:308] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] table processor_delay.t17 exists table processor_delay.t18 not exists for 1-th check, retry later + set +x ./t/drop.test: ok! 48 test cases passed, take time 8.942987045 s time="2025-04-06T08:23:01+08:00" level=info msg="run test [drop] ok" time="2025-04-06T08:23:01+08:00" level=warning msg="Create new db&{{{} {} 0} 0xc0003100a0 {{} {} 0} {0 0} [0xc00020f880] {[]} 1 0xc0002b4620 false map[0xc00020f880:map[0xc00020f880:true]] map[] 0 0 0 0 0 0 0 0 0x4a1c00}" check diff failed 3-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/event_filter/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... table processor_resolved_ts_fallback.t3 exists cdc cli processor list --server http://127.0.0.1:8301 |jq '.|length'|grep -E '^2$' table ddl_manager.finish_mark exists table batch_update_to_no_batch.v1 exists table batch_update_to_no_batch.recover_and_insert not exists for 1-th check, retry later ***************** properties ***************** "readallfields"="true" "mysql.port"="4000" "mysql.user"="root" "threadcount"="10" "workload"="core" "recordcount"="50000" "readproportion"="0" "mysql.host"="127.0.0.1" "updateproportion"="0" "operationcount"="0" "mysql.db"="consistent_replicate_nfs" "insertproportion"="0" "requestdistribution"="uniform" "fieldcount"="3" "scanproportion"="0" "dotransactions"="false" ********************************************** + set +x + tso='457152654477885441 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 457152654477885441 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Sun Apr 6 08:23:02 CST 2025] <<<<<< START cdc server in resourcecontrol case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.resourcecontrol.1171611718.out server --log-file /tmp/tidb_cdc_test/resourcecontrol/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/resourcecontrol/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 2 run task successfully killing cdc1: 5615 cdc cli capture list --server http://127.0.0.1:8302 |jq '.|length'|grep -E '^1$' check diff successfully ./t/mysql_replace.test: ok! 16 test cases passed, take time 1.044251866 s time="2025-04-06T08:23:02+08:00" level=info msg="run test [mysql_replace] ok" Great, All tests passed mysqltest end Error: Get "http://127.0.0.1:8301/api/v2/captures": dial tcp 127.0.0.1:8301: connect: connection refused run task failed 1-th time, retry later table processor_delay.t18 exists table processor_delay.t19 not exists for 1-th check, retry later wait process cdc.test exit for 1-th time... cdc.test cli capture list 2>&1 | grep '"is-owner": true' table test.finish_mark not exists for 1-th check, retry later [2025/04/06 08:23:03.827 +08:00] [INFO] [case.go:544] ["upstream create table executed"] [tblName=finishmark10000] [2025/04/06 08:23:03.827 +08:00] [INFO] [case.go:576] ["downstream start wait for table"] [tblName=finishmark10000] wait process cdc.test exit for 2-th time... check diff failed 4-th time, retry later table batch_update_to_no_batch.recover_and_insert exists table batch_update_to_no_batch.finish_mark not exists for 1-th check, retry later wait process cdc.test exit for 3-th time... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489989068044597362 TiDB Cluster ID. tikv_gc_leader_uuid 65821eae0e40003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-x-4tfw6, pid:6665, start at 2025-04-06 08:23:02.460878073 +0800 CST m=+2.585635388 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:25:02.470 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:23:02.457 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:13:02.457 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489989068044597362 TiDB Cluster ID. tikv_gc_leader_uuid 65821eae0e40003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-x-4tfw6, pid:6665, start at 2025-04-06 08:23:02.460878073 +0800 CST m=+2.585635388 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:25:02.470 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:23:02.457 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:13:02.457 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489989071369976476 TiDB Cluster ID. tikv_gc_leader_uuid 65821eae0800007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-x-4tfw6, pid:6740, start at 2025-04-06 08:23:02.469055644 +0800 CST m=+2.538389982 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:25:02.477 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:23:02.481 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:13:02.481 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/common_1/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/common_1/tiflash/log/error.log arg matches is ArgMatches { args: {"memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/common_1/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/common_1/tiflash/db/proxy"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/common_1/tiflash/log/proxy.log"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } wait process cdc.test exit for 4-th time... start tidb cluster in /tmp/tidb_cdc_test/event_filter Starting Upstream PD... cdc cli capture list --server http://127.0.0.1:8302 |jq '.|length'|grep -E '^1$' Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... Error: Get "http://127.0.0.1:8301/api/v2/captures": dial tcp 127.0.0.1:8301: connect: connection refused run task failed 2-th time, retry later table processor_delay.t19 exists table processor_delay.t20 not exists for 1-th check, retry later cdc.test: no process found wait process cdc.test exit for 5-th time... process cdc.test already exit [Sun Apr 6 08:23:05 CST 2025] <<<<<< run test case ddl_manager success! >>>>>> check diff failed 5-th time, retry later check_changefeed_state http://127.0.0.1:2379 b7be638f-3fc4-4ff3-ba27-e59a1f77064f finished null + endpoints=http://127.0.0.1:2379 + changefeed_id=b7be638f-3fc4-4ff3-ba27-e59a1f77064f + expected_state=finished + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c b7be638f-3fc4-4ff3-ba27-e59a1f77064f -s + info='{ "upstream_id": 7489988621899277058, "namespace": "default", "id": "b7be638f-3fc4-4ff3-ba27-e59a1f77064f", "state": "finished", "checkpoint_tso": 457152652707102726, "checkpoint_time": "2025-04-06 08:22:54.255", "error": null }' + echo '{ "upstream_id": 7489988621899277058, "namespace": "default", "id": "b7be638f-3fc4-4ff3-ba27-e59a1f77064f", "state": "finished", "checkpoint_tso": 457152652707102726, "checkpoint_time": "2025-04-06 08:22:54.255", "error": null }' { "upstream_id": 7489988621899277058, "namespace": "default", "id": "b7be638f-3fc4-4ff3-ba27-e59a1f77064f", "state": "finished", "checkpoint_tso": 457152652707102726, "checkpoint_time": "2025-04-06 08:22:54.255", "error": null } ++ echo '{' '"upstream_id":' 7489988621899277058, '"namespace":' '"default",' '"id":' '"b7be638f-3fc4-4ff3-ba27-e59a1f77064f",' '"state":' '"finished",' '"checkpoint_tso":' 457152652707102726, '"checkpoint_time":' '"2025-04-06' '08:22:54.255",' '"error":' null '}' ++ jq -r .state + state=finished + [[ ! finished == \f\i\n\i\s\h\e\d ]] ++ echo '{' '"upstream_id":' 7489988621899277058, '"namespace":' '"default",' '"id":' '"b7be638f-3fc4-4ff3-ba27-e59a1f77064f",' '"state":' '"finished",' '"checkpoint_tso":' 457152652707102726, '"checkpoint_time":' '"2025-04-06' '08:22:54.255",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Sun Apr 6 08:23:00 CST 2025] <<<<<< run test case changefeed_finish success! >>>>>> + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 06 Apr 2025 00:23:05 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7bae4ca1-957d-49f3-beb8-3fdd80f10629 {"id":"7bae4ca1-957d-49f3-beb8-3fdd80f10629","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898982} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087a827fe2 7bae4ca1-957d-49f3-beb8-3fdd80f10629 /tidb/cdc/default/default/upstream/7489989052176982054 {"id":7489989052176982054,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7bae4ca1-957d-49f3-beb8-3fdd80f10629 {"id":"7bae4ca1-957d-49f3-beb8-3fdd80f10629","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898982} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087a827fe2 7bae4ca1-957d-49f3-beb8-3fdd80f10629 /tidb/cdc/default/default/upstream/7489989052176982054 {"id":7489989052176982054,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7bae4ca1-957d-49f3-beb8-3fdd80f10629 {"id":"7bae4ca1-957d-49f3-beb8-3fdd80f10629","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898982} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087a827fe2 7bae4ca1-957d-49f3-beb8-3fdd80f10629 /tidb/cdc/default/default/upstream/7489989052176982054 {"id":7489989052176982054,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.resourcecontrol.cli.11773.out cli changefeed create --start-ts=457152654477885441 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ table test.finish_mark not exists for 2-th check, retry later Create changefeed successfully! ID: 48af0afa-47c6-4651-a07a-d81680eea5b3 Info: {"upstream_id":7489989052176982054,"namespace":"default","id":"48af0afa-47c6-4651-a07a-d81680eea5b3","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-06T08:23:06.014032803+08:00","start_ts":457152654477885441,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457152654477885441,"checkpoint_ts":457152654477885441,"checkpoint_time":"2025-04-06 08:23:01.010"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... [2025/04/06 08:23:05.827 +08:00] [INFO] [case.go:687] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark10000] [2025/04/06 08:23:05.836 +08:00] [INFO] [case.go:697] ["try parse finishedTs from ticdc log"] [tableName=finishmark10000] /tmp/tidb_cdc_test/bank/cdcdown.log [2025/04/06 08:23:05.838 +08:00] [INFO] [case.go:717] ["total files"] [file="[/tmp/tidb_cdc_test/bank/cdcdown.log]"] table batch_update_to_no_batch.finish_mark not exists for 2-th check, retry later found first match line: 457152655609036814 : [2025/04/06 08:23:06.426 +08:00] [INFO] [ddl_manager.go:298] ["handle a ddl job"] [namespace=default] [changefeed=changefeed-for-find-finished-ts] [tableID=154] [jobID=155] [query="CREATE TABLE IF NOT EXISTS `finishmark10000` (`foo` BIGINT PRIMARY KEY)"] [finishedTs=457152655609036814] [2025/04/06 08:23:06.831 +08:00] [INFO] [case.go:581] ["ddl synced"] [table=finishmark10000] [2025/04/06 08:23:06.831 +08:00] [INFO] [case.go:587] ["downstream sync success"] [endTs=457152655609036814] [2025/04/06 08:23:06.834 +08:00] [INFO] [case.go:191] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2025/04/06 08:23:06.838 +08:00] [INFO] [case.go:191] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2025/04/06 08:23:06.844 +08:00] [INFO] [case.go:308] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2025/04/06 08:23:06.929 +08:00] [INFO] [case.go:308] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory The 1 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory + set +x table processor_delay.t20 exists table processor_delay.t21 not exists for 1-th check, retry later table resourcecontrol.finish_mark not exists for 1-th check, retry later table test.finish_mark not exists for 3-th check, retry later check diff successfully [2025/04/06 08:23:08.437 +08:00] [INFO] [case.go:544] ["upstream create table executed"] [tblName=finishmark11000] [2025/04/06 08:23:08.438 +08:00] [INFO] [case.go:576] ["downstream start wait for table"] [tblName=finishmark11000] + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.common_1.cli.8068.out cli tso query --pd=http://127.0.0.1:2379 table batch_update_to_no_batch.finish_mark not exists for 3-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release check diff failed 1-th time, retry later cdc cli capture list --server http://127.0.0.1:8302 |jq '.|length'|grep -E '^1$' Error: Get "http://127.0.0.1:8301/api/v2/captures": dial tcp 127.0.0.1:8301: connect: connection refused run task failed 3-th time, retry later table processor_delay.t21 exists table processor_delay.t22 not exists for 1-th check, retry later table resourcecontrol.finish_mark exists table test.finish_mark not exists for 4-th check, retry later check diff successfully + set +x + tso='457152656477257729 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 457152656477257729 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Sun Apr 6 08:23:10 CST 2025] <<<<<< START cdc server in common_1 case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.common_1.81118113.out server --log-file /tmp/tidb_cdc_test/common_1/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/common_1/cdc_data --cluster-id default + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 wait process cdc.test exit for 1-th time... Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore table batch_update_to_no_batch.finish_mark not exists for 4-th check, retry later chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/batch_add_table Starting Upstream PD... Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... wait process cdc.test exit for 2-th time... [2025/04/06 08:23:10.525 +08:00] [INFO] [case.go:687] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark11000] [2025/04/06 08:23:10.538 +08:00] [INFO] [case.go:697] ["try parse finishedTs from ticdc log"] [tableName=finishmark11000] /tmp/tidb_cdc_test/bank/cdcdown.log [2025/04/06 08:23:10.626 +08:00] [INFO] [case.go:717] ["total files"] [file="[/tmp/tidb_cdc_test/bank/cdcdown.log]"] check diff failed 2-th time, retry later table processor_delay.t22 exists table processor_delay.t23 not exists for 1-th check, retry later "is-owner": true, run task successfully cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Sun Apr 6 08:23:11 CST 2025] <<<<<< run test case resourcecontrol success! >>>>>> found first match line: 457152656814899206 : [2025/04/06 08:23:10.264 +08:00] [INFO] [ddl_manager.go:298] ["handle a ddl job"] [namespace=default] [changefeed=changefeed-for-find-finished-ts] [tableID=156] [jobID=157] [query="CREATE TABLE IF NOT EXISTS `finishmark11000` (`foo` BIGINT PRIMARY KEY)"] [finishedTs=457152656814899206] [2025/04/06 08:23:11.440 +08:00] [INFO] [case.go:581] ["ddl synced"] [table=finishmark11000] [2025/04/06 08:23:11.440 +08:00] [INFO] [case.go:587] ["downstream sync success"] [endTs=457152656814899206] [2025/04/06 08:23:11.443 +08:00] [INFO] [case.go:191] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2025/04/06 08:23:11.446 +08:00] [INFO] [case.go:191] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2025/04/06 08:23:11.451 +08:00] [INFO] [case.go:308] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2025/04/06 08:23:11.528 +08:00] [INFO] [case.go:308] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] table test.finish_mark not exists for 5-th check, retry later owner pid: 4316 owner id 11473620-8337-41a3-80cc-aea294cfac6f [Sun Apr 6 08:23:11 CST 2025] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/sleep-in-owner-tick=1*sleep(6000)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.44914493.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_gap_between_watch_capture.server2.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_gap_between_watch_capture.server2 --cluster-id default --addr 127.0.0.1:8301 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id: 1' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ check data failed 1-th time, retry later check data successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Sun Apr 6 08:23:09 CST 2025] <<<<<< run test case ddl_puller_lag success! >>>>>> =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/force_replicate_table/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... INSERT - Takes(s): 10.0, Count: 17228, OPS: 1723.6, Avg(us): 5776, Min(us): 840, Max(us): 92900, 95th(us): 54000, 99th(us): 89000 table batch_update_to_no_batch.finish_mark exists check diff successfully ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 1-th time... table processor_delay.t23 exists table processor_delay.t24 not exists for 1-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 06 Apr 2025 00:23:13 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ec95d0d5-badb-42a1-b801-c5d9cc0e374c {"id":"ec95d0d5-badb-42a1-b801-c5d9cc0e374c","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898990} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087a9816f5 ec95d0d5-badb-42a1-b801-c5d9cc0e374c /tidb/cdc/default/default/upstream/7489989068044597362 {"id":7489989068044597362,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ec95d0d5-badb-42a1-b801-c5d9cc0e374c {"id":"ec95d0d5-badb-42a1-b801-c5d9cc0e374c","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898990} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087a9816f5 ec95d0d5-badb-42a1-b801-c5d9cc0e374c /tidb/cdc/default/default/upstream/7489989068044597362 {"id":7489989068044597362,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ec95d0d5-badb-42a1-b801-c5d9cc0e374c {"id":"ec95d0d5-badb-42a1-b801-c5d9cc0e374c","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898990} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087a9816f5 ec95d0d5-badb-42a1-b801-c5d9cc0e374c /tidb/cdc/default/default/upstream/7489989068044597362 {"id":7489989068044597362,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: 162cfca2-9563-46bf-8e3d-e3794ef8982c Info: {"upstream_id":7489989068044597362,"namespace":"default","id":"162cfca2-9563-46bf-8e3d-e3794ef8982c","sink_uri":"mysql://root@127.0.0.1:3306/","create_time":"2025-04-06T08:23:13.279317595+08:00","start_ts":457152656477257729,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457152656477257729,"checkpoint_ts":457152656477257729,"checkpoint_time":"2025-04-06 08:23:08.637"} table test.finish_mark not exists for 6-th check, retry later check diff failed 3-th time, retry later wait process cdc.test exit for 2-th time... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Sun Apr 6 08:23:14 CST 2025] <<<<<< run test case batch_update_to_no_batch success! >>>>>> [2025/04/06 08:23:14.346 +08:00] [INFO] [case.go:544] ["upstream create table executed"] [tblName=finishmark12000] [2025/04/06 08:23:14.346 +08:00] [INFO] [case.go:576] ["downstream start wait for table"] [tblName=finishmark12000] table processor_delay.t24 exists table processor_delay.t25 not exists for 1-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489989119248330045 TiDB Cluster ID. tikv_gc_leader_uuid 65821eb8e30000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-j-shst4, pid:6510, start at 2025-04-06 08:23:13.57895734 +0800 CST m=+3.118204436 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:25:13.588 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:23:13.625 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:13:13.625 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489989119248330045 TiDB Cluster ID. tikv_gc_leader_uuid 65821eb8e30000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-j-shst4, pid:6510, start at 2025-04-06 08:23:13.57895734 +0800 CST m=+3.118204436 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:25:13.588 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:23:13.625 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:13:13.625 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489989115107398672 TiDB Cluster ID. tikv_gc_leader_uuid 65821eb90340002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-j-shst4, pid:6573, start at 2025-04-06 08:23:13.678720831 +0800 CST m=+3.139495132 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:25:13.739 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:23:13.727 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:13:13.727 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/event_filter/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/event_filter/tiflash/log/error.log arg matches is ArgMatches { args: {"memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/event_filter/tiflash/db/proxy"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/event_filter/tiflash/log/proxy.log"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/event_filter/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 06 Apr 2025 00:23:14 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/e684662d-022f-4a79-9ef3-0a59dba96193 {UpstreamID:7489988628380945915 Namespace:default ID:e684662d-022f-4a79-9ef3-0a59dba96193 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-06 08:21:28.032207553 +0800 CST StartTs:457152628763131907 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0037fd170 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457152630100066306} {CheckpointTs:457152657992187905 MinTableBarrierTs:457152657992187906 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0f0aa18e-585e-4e9c-8f63-7c275941d14a {"id":"0f0aa18e-585e-4e9c-8f63-7c275941d14a","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898992} /tidb/cdc/default/__cdc_meta__/capture/11473620-8337-41a3-80cc-aea294cfac6f {"id":"11473620-8337-41a3-80cc-aea294cfac6f","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898973} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 1 /tidb/cdc/default/__cdc_meta__/owner/223196087905496e 11473620-8337-41a3-80cc-aea294cfac6f /tidb/cdc/default/__cdc_meta__/owner/2231960879054a10 0f0aa18e-585e-4e9c-8f63-7c275941d14a /tidb/cdc/default/default/changefeed/info/e684662d-022f-4a79-9ef3-0a59dba96193 {"upstream-id":7489988628380945915,"namespace":"default","changefeed-id":"e684662d-022f-4a79-9ef3-0a59dba96193","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-06T08:21:28.032207553+08:00","start-ts":457152628763131907,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457152630100066306} /tidb/cdc/default/default/changefeed/status/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":457152657992187905,"min-table-barrier-ts":457152657992187906,"admin-job-type":0} /tidb/cdc/default/default/task/position/0f0aa18e-585e-4e9c-8f63-7c275941d14a/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/11473620-8337-41a3-80cc-aea294cfac6f/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489988628380945915 {"id":7489988628380945915,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/e684662d-022f-4a79-9ef3-0a59dba96193 {UpstreamID:7489988628380945915 Namespace:default ID:e684662d-022f-4a79-9ef3-0a59dba96193 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-06 08:21:28.032207553 +0800 CST StartTs:457152628763131907 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0037fd170 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457152630100066306} {CheckpointTs:457152657992187905 MinTableBarrierTs:457152657992187906 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0f0aa18e-585e-4e9c-8f63-7c275941d14a {"id":"0f0aa18e-585e-4e9c-8f63-7c275941d14a","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898992} /tidb/cdc/default/__cdc_meta__/capture/11473620-8337-41a3-80cc-aea294cfac6f {"id":"11473620-8337-41a3-80cc-aea294cfac6f","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898973} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 1 /tidb/cdc/default/__cdc_meta__/owner/223196087905496e 11473620-8337-41a3-80cc-aea294cfac6f /tidb/cdc/default/__cdc_meta__/owner/2231960879054a10 0f0aa18e-585e-4e9c-8f63-7c275941d14a /tidb/cdc/default/default/changefeed/info/e684662d-022f-4a79-9ef3-0a59dba96193 {"upstream-id":7489988628380945915,"namespace":"default","changefeed-id":"e684662d-022f-4a79-9ef3-0a59dba96193","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-06T08:21:28.032207553+08:00","start-ts":457152628763131907,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457152630100066306} /tidb/cdc/default/default/changefeed/status/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":457152657992187905,"min-table-barrier-ts":457152657992187906,"admin-job-type":0} /tidb/cdc/default/default/task/position/0f0aa18e-585e-4e9c-8f63-7c275941d14a/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/11473620-8337-41a3-80cc-aea294cfac6f/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489988628380945915 {"id":7489988628380945915,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + grep -q 'etcd info' + echo ' *** processors info ***: changefeedID: default/e684662d-022f-4a79-9ef3-0a59dba96193 {UpstreamID:7489988628380945915 Namespace:default ID:e684662d-022f-4a79-9ef3-0a59dba96193 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-06 08:21:28.032207553 +0800 CST StartTs:457152628763131907 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0037fd170 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457152630100066306} {CheckpointTs:457152657992187905 MinTableBarrierTs:457152657992187906 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0f0aa18e-585e-4e9c-8f63-7c275941d14a {"id":"0f0aa18e-585e-4e9c-8f63-7c275941d14a","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898992} /tidb/cdc/default/__cdc_meta__/capture/11473620-8337-41a3-80cc-aea294cfac6f {"id":"11473620-8337-41a3-80cc-aea294cfac6f","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898973} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 1 /tidb/cdc/default/__cdc_meta__/owner/223196087905496e 11473620-8337-41a3-80cc-aea294cfac6f /tidb/cdc/default/__cdc_meta__/owner/2231960879054a10 0f0aa18e-585e-4e9c-8f63-7c275941d14a /tidb/cdc/default/default/changefeed/info/e684662d-022f-4a79-9ef3-0a59dba96193 {"upstream-id":7489988628380945915,"namespace":"default","changefeed-id":"e684662d-022f-4a79-9ef3-0a59dba96193","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-06T08:21:28.032207553+08:00","start-ts":457152628763131907,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457152630100066306} /tidb/cdc/default/default/changefeed/status/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":457152657992187905,"min-table-barrier-ts":457152657992187906,"admin-job-type":0} /tidb/cdc/default/default/task/position/0f0aa18e-585e-4e9c-8f63-7c275941d14a/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/11473620-8337-41a3-80cc-aea294cfac6f/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489988628380945915 {"id":7489988628380945915,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list 2>&1 | grep -v "11473620-8337-41a3-80cc-aea294cfac6f" | grep id check diff failed 4-th time, retry later cdc cli capture list --server http://127.0.0.1:8302 |jq '.|length'|grep -E '^1$' 1 run task successfully check diff failed 1-th time, retry later table test.finish_mark not exists for 7-th check, retry later Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/force_replicate_table Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... "cluster-id": "default" "id": "0f0aa18e-585e-4e9c-8f63-7c275941d14a", "cluster-id": "default" run task successfully [2025/04/06 08:23:16.346 +08:00] [INFO] [case.go:687] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark12000] [2025/04/06 08:23:16.427 +08:00] [INFO] [case.go:697] ["try parse finishedTs from ticdc log"] [tableName=finishmark12000] /tmp/tidb_cdc_test/bank/cdcdown.log [2025/04/06 08:23:16.429 +08:00] [INFO] [case.go:717] ["total files"] [file="[/tmp/tidb_cdc_test/bank/cdcdown.log]"] [Sun Apr 6 08:23:16 CST 2025] <<<<<< START cdc server in event_filter case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.event_filter.78127814.out server --log-file /tmp/tidb_cdc_test/event_filter/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/event_filter/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 found first match line: 457152658151833614 : [2025/04/06 08:23:15.264 +08:00] [INFO] [ddl_manager.go:298] ["handle a ddl job"] [namespace=default] [changefeed=changefeed-for-find-finished-ts] [tableID=158] [jobID=159] [query="CREATE TABLE IF NOT EXISTS `finishmark12000` (`foo` BIGINT PRIMARY KEY)"] [finishedTs=457152658151833614] [2025/04/06 08:23:17.233 +08:00] [INFO] [case.go:581] ["ddl synced"] [table=finishmark12000] [2025/04/06 08:23:17.233 +08:00] [INFO] [case.go:587] ["downstream sync success"] [endTs=457152658151833614] [2025/04/06 08:23:17.237 +08:00] [INFO] [case.go:191] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2025/04/06 08:23:17.241 +08:00] [INFO] [case.go:191] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2025/04/06 08:23:17.247 +08:00] [INFO] [case.go:308] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2025/04/06 08:23:17.253 +08:00] [INFO] [case.go:308] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] check diff successfully =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/multi_source/run.sh using Sink-Type: mysql... <<================= table processor_delay.t25 exists table processor_delay.t26 not exists for 1-th check, retry later check diff successfully table test.finish_mark not exists for 8-th check, retry later capture_id: 0f0aa18e-585e-4e9c-8f63-7c275941d14a wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 1-th time, retry later cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Sun Apr 6 08:23:18 CST 2025] <<<<<< run test case processor_resolved_ts_fallback success! >>>>>> table processor_delay.t26 exists table processor_delay.t27 not exists for 1-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table test.finish_mark not exists for 9-th check, retry later [2025/04/06 08:23:19.538 +08:00] [INFO] [case.go:544] ["upstream create table executed"] [tblName=finishmark13000] [2025/04/06 08:23:19.538 +08:00] [INFO] [case.go:576] ["downstream start wait for table"] [tblName=finishmark13000] =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/ddl_only_block_related_table/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 06 Apr 2025 00:23:20 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/043bd0d6-1e81-46b5-a89f-129c69041c4c {"id":"043bd0d6-1e81-46b5-a89f-129c69041c4c","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898997} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087ac188d4 043bd0d6-1e81-46b5-a89f-129c69041c4c /tidb/cdc/default/default/upstream/7489989119248330045 {"id":7489989119248330045,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/043bd0d6-1e81-46b5-a89f-129c69041c4c {"id":"043bd0d6-1e81-46b5-a89f-129c69041c4c","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898997} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087ac188d4 043bd0d6-1e81-46b5-a89f-129c69041c4c /tidb/cdc/default/default/upstream/7489989119248330045 {"id":7489989119248330045,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/043bd0d6-1e81-46b5-a89f-129c69041c4c {"id":"043bd0d6-1e81-46b5-a89f-129c69041c4c","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743898997} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087ac188d4 043bd0d6-1e81-46b5-a89f-129c69041c4c /tidb/cdc/default/default/upstream/7489989119248330045 {"id":7489989119248330045,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.event_filter.cli.7874.out cli changefeed create --sink-uri=mysql://normal:123456@127.0.0.1:3306/ --server=127.0.0.1:8300 --config=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/event_filter/conf/cf.toml VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489989139536271564 TiDB Cluster ID. tikv_gc_leader_uuid 65821ebdf240009 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-x-hbn5c, pid:6678, start at 2025-04-06 08:23:18.75565466 +0800 CST m=+2.816502742 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:25:18.763 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:23:18.729 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:13:18.729 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489989139536271564 TiDB Cluster ID. tikv_gc_leader_uuid 65821ebdf240009 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-x-hbn5c, pid:6678, start at 2025-04-06 08:23:18.75565466 +0800 CST m=+2.816502742 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:25:18.763 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:23:18.729 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:13:18.729 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489989141324544638 TiDB Cluster ID. tikv_gc_leader_uuid 65821ebdfb8000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-x-hbn5c, pid:6750, start at 2025-04-06 08:23:18.793660534 +0800 CST m=+2.794389035 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:25:18.838 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:23:18.816 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:13:18.816 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/batch_add_table/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/batch_add_table/tiflash/log/error.log arg matches is ArgMatches { args: {"config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/batch_add_table/tiflash-proxy.toml"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/batch_add_table/tiflash/db/proxy"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/batch_add_table/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } check diff failed 2-th time, retry later Create changefeed successfully! ID: 328cdde6-4a8a-4719-a3f6-0a90b447cb96 Info: {"upstream_id":7489989119248330045,"namespace":"default","id":"328cdde6-4a8a-4719-a3f6-0a90b447cb96","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-06T08:23:20.568106422+08:00","start_ts":457152659573178371,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["event_filter.*"],"event_filters":[{"matcher":["event_filter.t1"],"ignore_event":["drop table","delete"],"ignore_sql":null,"ignore_insert_value_expr":"id = 2 or city = 'tokyo'","ignore_update_new_value_expr":"","ignore_update_old_value_expr":"","ignore_delete_value_expr":""},{"matcher":["event_filter.t_truncate"],"ignore_event":["truncate table"],"ignore_sql":null,"ignore_insert_value_expr":"","ignore_update_new_value_expr":"","ignore_update_old_value_expr":"","ignore_delete_value_expr":""},{"matcher":["event_filter.t_alter"],"ignore_event":["alter table"],"ignore_sql":null,"ignore_insert_value_expr":"","ignore_update_new_value_expr":"","ignore_update_old_value_expr":"","ignore_delete_value_expr":""},{"matcher":["event_filter.t_name*"],"ignore_event":["rename table"],"ignore_sql":null,"ignore_insert_value_expr":"","ignore_update_new_value_expr":"","ignore_update_old_value_expr":"","ignore_delete_value_expr":""}]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457152659573178371,"checkpoint_ts":457152659573178371,"checkpoint_time":"2025-04-06 08:23:20.447"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... nonempty select id, val from test.availability1 where id=1 and val=1 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later table processor_delay.t27 exists table processor_delay.t28 not exists for 1-th check, retry later table common_1.v1 exists table common_1.recover_and_insert not exists for 1-th check, retry later The 1 times to try to start tidb cluster... Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.finish_mark not exists for 10-th check, retry later [2025/04/06 08:23:21.538 +08:00] [INFO] [case.go:687] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark13000] [2025/04/06 08:23:21.629 +08:00] [INFO] [case.go:697] ["try parse finishedTs from ticdc log"] [tableName=finishmark13000] /tmp/tidb_cdc_test/bank/cdcdown.log [2025/04/06 08:23:21.631 +08:00] [INFO] [case.go:717] ["total files"] [file="[/tmp/tidb_cdc_test/bank/cdcdown.log]"] + set +x INSERT - Takes(s): 20.1, Count: 28626, OPS: 1427.0, Avg(us): 6961, Min(us): 840, Max(us): 95017, 95th(us): 78000, 99th(us): 88000 table event_filter.t1 does not exists table event_filter.t1 not exists for 1-th check, retry later found first match line: 457152659514982413 : [2025/04/06 08:23:20.264 +08:00] [INFO] [ddl_manager.go:298] ["handle a ddl job"] [namespace=default] [changefeed=changefeed-for-find-finished-ts] [tableID=160] [jobID=161] [query="CREATE TABLE IF NOT EXISTS `finishmark13000` (`foo` BIGINT PRIMARY KEY)"] [finishedTs=457152659514982413] [2025/04/06 08:23:22.532 +08:00] [INFO] [case.go:581] ["ddl synced"] [table=finishmark13000] [2025/04/06 08:23:22.532 +08:00] [INFO] [case.go:587] ["downstream sync success"] [endTs=457152659514982413] [2025/04/06 08:23:22.535 +08:00] [INFO] [case.go:191] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] table common_1.recover_and_insert not exists for 2-th check, retry later [2025/04/06 08:23:22.632 +08:00] [INFO] [case.go:191] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2025/04/06 08:23:22.638 +08:00] [INFO] [case.go:308] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2025/04/06 08:23:22.733 +08:00] [INFO] [case.go:308] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [Sun Apr 6 08:23:22 CST 2025] <<<<<< START cdc server in batch_add_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.batch_add_table.80858087.out server --log-file /tmp/tidb_cdc_test/batch_add_table/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/batch_add_table/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 nonempty select id, val from test.availability1 where id=1 and val=1 run task successfully nonempty select id, val from test.availability1 where id=1 and val=22 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later check diff failed 3-th time, retry later table processor_delay.t28 exists table processor_delay.t29 not exists for 1-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/ddl_only_block_related_table Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.finish_mark not exists for 11-th check, retry later \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } [Pipeline] // timeout [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-periodics_cdc_integration_mysql_test-234/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } table common_1.recover_and_insert not exists for 3-th check, retry later [2025/04/06 08:23:24.727 +08:00] [INFO] [case.go:544] ["upstream create table executed"] [tblName=finishmark14000] [2025/04/06 08:23:24.727 +08:00] [INFO] [case.go:576] ["downstream start wait for table"] [tblName=finishmark14000] [Pipeline] // withEnv [Pipeline] } nonempty select id, val from test.availability1 where id=1 and val=22 check diff failed 4-th time, retry later [Pipeline] // stage table event_filter.t1 exists table event_filter.t_normal not exists for 1-th check, retry later [Pipeline] } run task successfully empty select id, val from test.availability1 where id=1 TEST FAILED: OUTPUT CONTAINS 'id:' ____________________________________ *************************** 1. row *************************** id: 1 val: 22 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 06 Apr 2025 00:23:25 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/92599e2c-0e0d-4d5e-954b-da975a94f788 {"id":"92599e2c-0e0d-4d5e-954b-da975a94f788","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899002} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087ad8aed8 92599e2c-0e0d-4d5e-954b-da975a94f788 /tidb/cdc/default/default/upstream/7489989139536271564 {"id":7489989139536271564,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/92599e2c-0e0d-4d5e-954b-da975a94f788 {"id":"92599e2c-0e0d-4d5e-954b-da975a94f788","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899002} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087ad8aed8 92599e2c-0e0d-4d5e-954b-da975a94f788 /tidb/cdc/default/default/upstream/7489989139536271564 {"id":7489989139536271564,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/92599e2c-0e0d-4d5e-954b-da975a94f788 {"id":"92599e2c-0e0d-4d5e-954b-da975a94f788","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899002} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087ad8aed8 92599e2c-0e0d-4d5e-954b-da975a94f788 /tidb/cdc/default/default/upstream/7489989139536271564 {"id":7489989139536271564,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.batch_add_table.cli.8142.out cli changefeed create --sink-uri=mysql://normal:123456@127.0.0.1:3306/ --config=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/batch_add_table/conf/changefeed.toml start tidb cluster in /tmp/tidb_cdc_test/multi_source Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... table processor_delay.t29 exists table processor_delay.t30 not exists for 1-th check, retry later Create changefeed successfully! ID: 2a854446-d2d8-4609-acd8-3cae63e3beaa Info: {"upstream_id":7489989139536271564,"namespace":"default","id":"2a854446-d2d8-4609-acd8-3cae63e3beaa","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-06T08:23:25.864780733+08:00","start_ts":457152660969357315,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*","!test.t_1"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457152660969357315,"checkpoint_ts":457152660969357315,"checkpoint_time":"2025-04-06 08:23:25.773"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489989168206124187 TiDB Cluster ID. tikv_gc_leader_uuid 65821ec2f100012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-m-66zpf, pid:4063, start at 2025-04-06 08:23:23.871967488 +0800 CST m=+2.431000089 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:25:23.879 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:23:23.894 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:13:23.894 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489989168206124187 TiDB Cluster ID. tikv_gc_leader_uuid 65821ec2f100012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-m-66zpf, pid:4063, start at 2025-04-06 08:23:23.871967488 +0800 CST m=+2.431000089 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:25:23.879 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:23:23.894 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:13:23.894 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489989162868136431 TiDB Cluster ID. tikv_gc_leader_uuid 65821ec30800009 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-m-66zpf, pid:4128, start at 2025-04-06 08:23:23.951329738 +0800 CST m=+2.455462086 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:25:23.958 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:23:23.936 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:13:23.936 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/force_replicate_table/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/force_replicate_table/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/force_replicate_table/tiflash/log/proxy.log"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/force_replicate_table/tiflash/db/proxy"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/force_replicate_table/tiflash-proxy.toml"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table test.finish_mark not exists for 12-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/changefeed_resume_with_checkpoint_ts/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table common_1.recover_and_insert exists table common_1.finish_mark not exists for 1-th check, retry later check diff failed 5-th time, retry later table event_filter.t_normal exists table event_filter.t_truncate not exists for 1-th check, retry later empty select id, val from test.availability1 where id=1 run task successfully nonempty select id, val from test.availability2 where id=1 and val=1 [2025/04/06 08:23:26.727 +08:00] [INFO] [case.go:687] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark14000] [2025/04/06 08:23:26.737 +08:00] [INFO] [case.go:697] ["try parse finishedTs from ticdc log"] [tableName=finishmark14000] /tmp/tidb_cdc_test/bank/cdcdown.log [2025/04/06 08:23:26.739 +08:00] [INFO] [case.go:717] ["total files"] [file="[/tmp/tidb_cdc_test/bank/cdcdown.log]"] TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later + set +x found first match line: 457152660847722510 : [2025/04/06 08:23:26.364 +08:00] [INFO] [ddl_manager.go:298] ["handle a ddl job"] [namespace=default] [changefeed=changefeed-for-find-finished-ts] [tableID=162] [jobID=163] [query="CREATE TABLE IF NOT EXISTS `finishmark14000` (`foo` BIGINT PRIMARY KEY)"] [finishedTs=457152660847722510] [2025/04/06 08:23:27.435 +08:00] [INFO] [case.go:581] ["ddl synced"] [table=finishmark14000] [2025/04/06 08:23:27.435 +08:00] [INFO] [case.go:587] ["downstream sync success"] [endTs=457152660847722510] [2025/04/06 08:23:27.438 +08:00] [INFO] [case.go:191] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2025/04/06 08:23:27.443 +08:00] [INFO] [case.go:191] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2025/04/06 08:23:27.447 +08:00] [INFO] [case.go:308] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2025/04/06 08:23:27.453 +08:00] [INFO] [case.go:308] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] Verifying downstream PD is started... table test.t_1 not exists for 1-th check, retry later table processor_delay.t30 exists table processor_delay.t31 not exists for 1-th check, retry later [Sun Apr 6 08:23:27 CST 2025] <<<<<< START cdc server in force_replicate_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.force_replicate_table.55375539.out server --log-file /tmp/tidb_cdc_test/force_replicate_table/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/force_replicate_table/cdc_data --cluster-id default + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table test.finish_mark not exists for 13-th check, retry later Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table event_filter.t_truncate exists table event_filter.t_alter not exists for 1-th check, retry later Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } [Pipeline] // timeout table common_1.finish_mark not exists for 2-th check, retry later [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-periodics_cdc_integration_mysql_test-234/tiflow-cdc already exists) nonempty select id, val from test.availability2 where id=1 and val=1 run task successfully nonempty select id, val from test.availability2 where id=1 and val=22 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later [Pipeline] // cache [Pipeline] } check diff failed 6-th time, retry later [Pipeline] // dir [Pipeline] } [2025/04/06 08:23:21.560 +08:00] [WARN] [diff.go:180] ["table struct is not equal"] [reason="column num not equal, one is 2 another is 5"] [Pipeline] // stage table processor_delay.t31 exists table processor_delay.t32 not exists for 1-th check, retry later [Pipeline] } start tidb cluster in /tmp/tidb_cdc_test/changefeed_resume_with_checkpoint_ts Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } table test.finish_mark not exists for 14-th check, retry later [Pipeline] // node [Pipeline] } [2025/04/06 08:23:29.946 +08:00] [INFO] [case.go:544] ["upstream create table executed"] [tblName=finishmark15000] [2025/04/06 08:23:29.946 +08:00] [INFO] [case.go:576] ["downstream start wait for table"] [tblName=finishmark15000] [Pipeline] // podTemplate [Pipeline] } table test.t_1 exists table test.t_2 exists table test.t_3 exists table test.t_4 exists table test.t_5 exists table test.t_6 exists table test.t_7 exists table test.t_8 exists table test.t_9 exists table test.t_10 exists table test.t_11 exists table test.t_12 exists table test.t_13 exists table test.t_14 exists table test.t_15 exists table test.t_16 exists table test.t_17 exists table test.t_18 exists table test.t_19 exists table test.t_20 exists table test.t_21 exists table test.t_22 exists table test.t_23 exists table test.t_24 exists table test.t_25 exists table test.t_26 exists table test.t_27 exists table test.t_28 exists table test.t_29 exists table test.t_30 exists table test.t_31 exists table test.t_32 exists table test.t_33 exists table test.t_34 exists table test.t_35 exists table test.t_36 exists table test.t_37 exists table test.t_38 exists table test.t_39 exists table test.t_40 exists table test.t_41 exists table test.t_42 exists table test.t_43 exists table test.t_44 exists table test.t_45 exists table test.t_46 exists table test.t_47 exists table test.t_48 exists table test.t_49 exists table test.t_50 exists table test.t_51 exists table test.t_52 exists table test.t_53 exists table test.t_54 exists table test.t_55 exists table test.t_56 exists table test.t_57 exists table test.t_58 exists table test.t_59 exists [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.t_60 exists table test.t_61 exists table test.t_62 exists table test.t_63 exists table test.t_64 exists table test.t_65 exists table test.t_66 exists table test.t_67 exists table test.t_68 exists table test.t_69 exists table test.t_70 exists table test.t_71 exists table test.t_72 exists table test.t_73 exists table test.t_74 exists table test.t_75 exists table test.t_76 exists table test.t_77 exists table test.t_78 exists table test.t_79 exists table test.t_80 exists table test.t_81 exists table test.t_82 exists table test.t_83 exists table test.t_84 exists table test.t_85 exists table test.t_86 exists table test.t_87 exists table test.t_88 exists table event_filter.t_alter exists table event_filter.t_name not exists for 1-th check, retry later table test.t_89 exists table test.t_90 exists table test.t_91 exists table test.t_92 exists table test.t_93 exists table test.t_94 exists table test.t_95 exists table test.t_96 exists table test.t_97 exists table test.t_98 exists table test.t_99 exists table test.t_100 exists Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table common_1.finish_mark not exists for 3-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 06 Apr 2025 00:23:30 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b555a98a-64f4-464c-9b48-93a67aa8fa66 {"id":"b555a98a-64f4-464c-9b48-93a67aa8fa66","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899008} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087aec7bd7 b555a98a-64f4-464c-9b48-93a67aa8fa66 /tidb/cdc/default/default/upstream/7489989168206124187 {"id":7489989168206124187,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b555a98a-64f4-464c-9b48-93a67aa8fa66 {"id":"b555a98a-64f4-464c-9b48-93a67aa8fa66","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899008} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087aec7bd7 b555a98a-64f4-464c-9b48-93a67aa8fa66 /tidb/cdc/default/default/upstream/7489989168206124187 {"id":7489989168206124187,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b555a98a-64f4-464c-9b48-93a67aa8fa66 {"id":"b555a98a-64f4-464c-9b48-93a67aa8fa66","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899008} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087aec7bd7 b555a98a-64f4-464c-9b48-93a67aa8fa66 /tidb/cdc/default/default/upstream/7489989168206124187 {"id":7489989168206124187,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: 9cc06fda-d59c-4857-9ed7-e49a67e0b4b1 Info: {"upstream_id":7489989168206124187,"namespace":"default","id":"9cc06fda-d59c-4857-9ed7-e49a67e0b4b1","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/?safe-mode=true","create_time":"2025-04-06T08:23:31.13031605+08:00","start_ts":457152661511995399,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":true,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457152661511995399,"checkpoint_ts":457152661511995399,"checkpoint_time":"2025-04-06 08:23:27.843"} check diff successfully table processor_delay.t32 exists table processor_delay.t33 not exists for 1-th check, retry later table test.t_100 not exists for 1-th check, retry later table test.finish_mark not exists for 15-th check, retry later nonempty select id, val from test.availability2 where id=1 and val=22 run task successfully empty select id, val from test.availability2 where id=1 TEST FAILED: OUTPUT CONTAINS 'id:' ____________________________________ *************************** 1. row *************************** id: 1 val: 22 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later Verifying downstream PD is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489989193297080848 TiDB Cluster ID. tikv_gc_leader_uuid 65821ec993c000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-c-m09xf, pid:6724, start at 2025-04-06 08:23:30.673917753 +0800 CST m=+2.616572433 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:25:30.682 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:23:30.688 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:13:30.688 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489989193297080848 TiDB Cluster ID. tikv_gc_leader_uuid 65821ec993c000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-c-m09xf, pid:6724, start at 2025-04-06 08:23:30.673917753 +0800 CST m=+2.616572433 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:25:30.682 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:23:30.688 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:13:30.688 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489989194506565990 TiDB Cluster ID. tikv_gc_leader_uuid 65821ec9b2c000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-c-m09xf, pid:6798, start at 2025-04-06 08:23:30.793405014 +0800 CST m=+2.646254916 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:25:30.840 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:23:30.829 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:13:30.829 +0800 All versions after safe point can be accessed. (DO NOT EDIT) INSERT - Takes(s): 30.0, Count: 40644, OPS: 1355.0, Avg(us): 7357, Min(us): 824, Max(us): 95760, 95th(us): 79000, 99th(us): 88000 Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/ddl_only_block_related_table/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/ddl_only_block_related_table/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/ddl_only_block_related_table/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/ddl_only_block_related_table/tiflash/db/proxy"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/ddl_only_block_related_table/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [2025/04/06 08:23:32.025 +08:00] [INFO] [case.go:687] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark15000] [2025/04/06 08:23:32.033 +08:00] [INFO] [case.go:697] ["try parse finishedTs from ticdc log"] [tableName=finishmark15000] /tmp/tidb_cdc_test/bank/cdcdown.log [2025/04/06 08:23:32.035 +08:00] [INFO] [case.go:717] ["total files"] [file="[/tmp/tidb_cdc_test/bank/cdcdown.log]"] check diff failed 1-th time, retry later table common_1.finish_mark not exists for 4-th check, retry later Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table event_filter.t_name exists table event_filter.t_name1 not exists for 1-th check, retry later table processor_delay.t33 exists table processor_delay.t34 not exists for 1-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) found first match line: 457152662473015310 : [2025/04/06 08:23:32.426 +08:00] [INFO] [ddl_manager.go:298] ["handle a ddl job"] [namespace=default] [changefeed=changefeed-for-find-finished-ts] [tableID=164] [jobID=165] [query="CREATE TABLE IF NOT EXISTS `finishmark15000` (`foo` BIGINT PRIMARY KEY)"] [finishedTs=457152662473015310] [2025/04/06 08:23:33.143 +08:00] [INFO] [case.go:581] ["ddl synced"] [table=finishmark15000] [2025/04/06 08:23:33.143 +08:00] [INFO] [case.go:587] ["downstream sync success"] [endTs=457152662473015310] [2025/04/06 08:23:33.147 +08:00] [INFO] [case.go:191] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2025/04/06 08:23:33.225 +08:00] [INFO] [case.go:191] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2025/04/06 08:23:33.230 +08:00] [INFO] [case.go:308] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2025/04/06 08:23:33.237 +08:00] [INFO] [case.go:308] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] table test.t_100 not exists for 2-th check, retry later empty select id, val from test.availability2 where id=1 run task successfully nonempty select id, val from test.availability3 where id=1 and val=1 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later table test.finish_mark not exists for 16-th check, retry later check diff failed 2-th time, retry later table common_1.finish_mark exists check diff successfully Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Sun Apr 6 08:23:34 CST 2025] <<<<<< START cdc server in ddl_only_block_related_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_only_block_related_table.80578059.out server --log-file /tmp/tidb_cdc_test/ddl_only_block_related_table/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_only_block_related_table/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table event_filter.t_name1 exists table event_filter.t_name2 not exists for 1-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489989201638240482 TiDB Cluster ID. tikv_gc_leader_uuid 65821ecc97c0013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-v-hm9ds, pid:7012, start at 2025-04-06 08:23:33.762089531 +0800 CST m=+3.110920030 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:25:33.772 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:23:33.827 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:13:33.827 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489989201638240482 TiDB Cluster ID. tikv_gc_leader_uuid 65821ecc97c0013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-v-hm9ds, pid:7012, start at 2025-04-06 08:23:33.762089531 +0800 CST m=+3.110920030 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:25:33.772 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:23:33.827 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:13:33.827 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489989206297782609 TiDB Cluster ID. tikv_gc_leader_uuid 65821eccc980007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-v-hm9ds, pid:7075, start at 2025-04-06 08:23:33.944655208 +0800 CST m=+3.208992731 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:25:33.956 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:23:33.926 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:13:33.926 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/multi_source/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/multi_source/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/multi_source/tiflash/log/proxy.log"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/multi_source/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/multi_source/tiflash-proxy.toml"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [2025/04/06 08:23:35.044 +08:00] [INFO] [case.go:544] ["upstream create table executed"] [tblName=finishmark16000] [2025/04/06 08:23:35.044 +08:00] [INFO] [case.go:576] ["downstream start wait for table"] [tblName=finishmark16000] table test.t_100 not exists for 3-th check, retry later wait process cdc.test exit for 1-th time... nonempty select id, val from test.availability3 where id=1 and val=1 run task successfully nonempty select id, val from test.availability3 where id=1 and val=22 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later table processor_delay.t34 exists table processor_delay.t35 not exists for 1-th check, retry later wait process cdc.test exit for 2-th time... table test.finish_mark not exists for 17-th check, retry later cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Sun Apr 6 08:23:36 CST 2025] <<<<<< run test case common_1 success! >>>>>> table event_filter.t_name2 exists table event_filter.t_name3 not exists for 1-th check, retry later check diff failed 3-th time, retry later [2025/04/06 08:23:37.045 +08:00] [INFO] [case.go:687] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark16000] [2025/04/06 08:23:37.131 +08:00] [INFO] [case.go:697] ["try parse finishedTs from ticdc log"] [tableName=finishmark16000] /tmp/tidb_cdc_test/bank/cdcdown.log [2025/04/06 08:23:37.132 +08:00] [INFO] [case.go:717] ["total files"] [file="[/tmp/tidb_cdc_test/bank/cdcdown.log]"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > table processor_delay.t35 exists table processor_delay.t36 not exists for 1-th check, retry later + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_source.cli.8303.out cli tso query --pd=http://127.0.0.1:2379 < HTTP/1.1 200 OK < Date: Sun, 06 Apr 2025 00:23:37 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/dc6ceb24-492d-4824-ad50-3ee041d8d180 {"id":"dc6ceb24-492d-4824-ad50-3ee041d8d180","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899014} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087b080fd6 dc6ceb24-492d-4824-ad50-3ee041d8d180 /tidb/cdc/default/default/upstream/7489989193297080848 {"id":7489989193297080848,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/dc6ceb24-492d-4824-ad50-3ee041d8d180 {"id":"dc6ceb24-492d-4824-ad50-3ee041d8d180","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899014} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087b080fd6 dc6ceb24-492d-4824-ad50-3ee041d8d180 /tidb/cdc/default/default/upstream/7489989193297080848 {"id":7489989193297080848,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/dc6ceb24-492d-4824-ad50-3ee041d8d180 {"id":"dc6ceb24-492d-4824-ad50-3ee041d8d180","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899014} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087b080fd6 dc6ceb24-492d-4824-ad50-3ee041d8d180 /tidb/cdc/default/default/upstream/7489989193297080848 {"id":7489989193297080848,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_only_block_related_table.cli.8116.out cli changefeed create --sink-uri=mysql://root@127.0.0.1:3306/ -c=ddl-only-block-related-table table test.t_100 not exists for 4-th check, retry later nonempty select id, val from test.availability3 where id=1 and val=22 run task successfully empty select id, val from test.availability3 where id=1 TEST FAILED: OUTPUT CONTAINS 'id:' ____________________________________ *************************** 1. row *************************** id: 1 val: 22 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later table test.finish_mark not exists for 18-th check, retry later Create changefeed successfully! ID: ddl-only-block-related-table Info: {"upstream_id":7489989193297080848,"namespace":"default","id":"ddl-only-block-related-table","sink_uri":"mysql://root@127.0.0.1:3306/","create_time":"2025-04-06T08:23:37.983731343+08:00","start_ts":457152664145494017,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457152664145494017,"checkpoint_ts":457152664145494017,"checkpoint_time":"2025-04-06 08:23:37.889"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... found first match line: 457152663814144004 : [2025/04/06 08:23:37.525 +08:00] [INFO] [ddl_manager.go:298] ["handle a ddl job"] [namespace=default] [changefeed=changefeed-for-find-finished-ts] [tableID=166] [jobID=167] [query="CREATE TABLE IF NOT EXISTS `finishmark16000` (`foo` BIGINT PRIMARY KEY)"] [finishedTs=457152663814144004] [2025/04/06 08:23:38.431 +08:00] [INFO] [case.go:581] ["ddl synced"] [table=finishmark16000] [2025/04/06 08:23:38.431 +08:00] [INFO] [case.go:587] ["downstream sync success"] [endTs=457152663814144004] [2025/04/06 08:23:38.434 +08:00] [INFO] [case.go:191] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2025/04/06 08:23:38.437 +08:00] [INFO] [case.go:191] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2025/04/06 08:23:38.442 +08:00] [INFO] [case.go:308] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2025/04/06 08:23:38.448 +08:00] [INFO] [case.go:308] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] table event_filter.t_name3 exists check diff failed 4-th time, retry later + set +x + tso='457152664037752833 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 457152664037752833 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Sun Apr 6 08:23:38 CST 2025] <<<<<< START cdc server in multi_source case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_source.83458347.out server --log-file /tmp/tidb_cdc_test/multi_source/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_source/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table force_replicate_table.t0 exists table force_replicate_table.t1 exists table force_replicate_table.t2 exists table force_replicate_table.t3 not exists for 1-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489989219320878993 TiDB Cluster ID. tikv_gc_leader_uuid 65821ecff8c0004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-4-x87nv, pid:6607, start at 2025-04-06 08:23:37.192239599 +0800 CST m=+2.368164548 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:25:37.234 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:23:37.237 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:13:37.237 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489989219320878993 TiDB Cluster ID. tikv_gc_leader_uuid 65821ecff8c0004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-4-x87nv, pid:6607, start at 2025-04-06 08:23:37.192239599 +0800 CST m=+2.368164548 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:25:37.234 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:23:37.237 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:13:37.237 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489989219990400340 TiDB Cluster ID. tikv_gc_leader_uuid 65821ed01e00004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-4-x87nv, pid:6681, start at 2025-04-06 08:23:37.340090897 +0800 CST m=+2.461561345 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:25:37.348 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:23:37.336 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:13:37.336 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/changefeed_resume_with_checkpoint_ts/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/changefeed_resume_with_checkpoint_ts/tiflash/log/error.log arg matches is ArgMatches { args: {"memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/changefeed_resume_with_checkpoint_ts/tiflash/db/proxy"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/changefeed_resume_with_checkpoint_ts/tiflash-proxy.toml"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/changefeed_resume_with_checkpoint_ts/tiflash/log/proxy.log"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table processor_delay.t36 exists table processor_delay.t37 not exists for 1-th check, retry later + set +x table test.t_100 not exists for 5-th check, retry later empty select id, val from test.availability3 where id=1 run task successfully test_gap_between_watch_capture pass table test.finish_mark not exists for 19-th check, retry later wait process cdc.test exit for 1-th time... table event_filter.finish_mark exists check diff failed 1-th time, retry later table force_replicate_table.t3 exists table force_replicate_table.t4 not exists for 1-th check, retry later Run finished, takes 37.305223395s INSERT - Takes(s): 37.3, Count: 50000, OPS: 1340.5, Avg(us): 7428, Min(us): 824, Max(us): 95760, 95th(us): 80000, 99th(us): 88000 table consistent_replicate_nfs.usertable exists wait process cdc.test exit for 1-th time... table testSync.usertable exists table testSync.simple1 exists table testSync.simple2 exists wait process cdc.test exit for 2-th time... wait process cdc.test exit for 2-th time... check diff successfully [2025/04/06 08:23:40.640 +08:00] [INFO] [case.go:544] ["upstream create table executed"] [tblName=finishmark17000] [2025/04/06 08:23:40.641 +08:00] [INFO] [case.go:576] ["downstream start wait for table"] [tblName=finishmark17000] cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit run test case delete_owner_key [Sun Apr 6 08:23:41 CST 2025] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.49004902.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_gap_between_watch_capture.server1.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_gap_between_watch_capture.server1 --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 wait process cdc.test exit for 3-th time... table processor_delay.t37 exists table processor_delay.t38 not exists for 1-th check, retry later table test.t_100 not exists for 6-th check, retry later table test.finish_mark not exists for 20-th check, retry later cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Sun Apr 6 08:23:41 CST 2025] <<<<<< START cdc server in changefeed_resume_with_checkpoint_ts case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_resume_with_checkpoint_ts.80178019.out server --log-file /tmp/tidb_cdc_test/changefeed_resume_with_checkpoint_ts/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_resume_with_checkpoint_ts/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table force_replicate_table.t4 exists table force_replicate_table.t5 not exists for 1-th check, retry later check diff failed 1-th time, retry later table ddl_only_block_related_table.finish_mark not exists for 1-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 06 Apr 2025 00:23:42 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/88d8f1c4-468b-447a-b9a5-670e62b5490e {"id":"88d8f1c4-468b-447a-b9a5-670e62b5490e","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899019} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087b1216e8 88d8f1c4-468b-447a-b9a5-670e62b5490e /tidb/cdc/default/default/upstream/7489989201638240482 {"id":7489989201638240482,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/88d8f1c4-468b-447a-b9a5-670e62b5490e {"id":"88d8f1c4-468b-447a-b9a5-670e62b5490e","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899019} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087b1216e8 88d8f1c4-468b-447a-b9a5-670e62b5490e /tidb/cdc/default/default/upstream/7489989201638240482 {"id":7489989201638240482,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/88d8f1c4-468b-447a-b9a5-670e62b5490e {"id":"88d8f1c4-468b-447a-b9a5-670e62b5490e","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899019} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087b1216e8 88d8f1c4-468b-447a-b9a5-670e62b5490e /tidb/cdc/default/default/upstream/7489989201638240482 {"id":7489989201638240482,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_source.cli.8403.out cli changefeed create --start-ts=457152664037752833 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ Create changefeed successfully! ID: 8aa5ed1a-e1e1-4275-897d-968831a6a2d7 Info: {"upstream_id":7489989201638240482,"namespace":"default","id":"8aa5ed1a-e1e1-4275-897d-968831a6a2d7","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-06T08:23:42.494362951+08:00","start_ts":457152664037752833,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457152664037752833,"checkpoint_ts":457152664037752833,"checkpoint_time":"2025-04-06 08:23:37.478"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... check diff failed 2-th time, retry later [2025/04/06 08:23:42.725 +08:00] [INFO] [case.go:687] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark17000] [2025/04/06 08:23:42.732 +08:00] [INFO] [case.go:697] ["try parse finishedTs from ticdc log"] [tableName=finishmark17000] /tmp/tidb_cdc_test/bank/cdcdown.log [2025/04/06 08:23:42.734 +08:00] [INFO] [case.go:717] ["total files"] [file="[/tmp/tidb_cdc_test/bank/cdcdown.log]"] table processor_delay.t38 exists table test.t_100 not exists for 7-th check, retry later table processor_delay.t39 exists table processor_delay.t40 not exists for 1-th check, retry later table test.finish_mark not exists for 21-th check, retry later found first match line: 457152665072435213 : [2025/04/06 08:23:42.425 +08:00] [INFO] [ddl_manager.go:298] ["handle a ddl job"] [namespace=default] [changefeed=changefeed-for-find-finished-ts] [tableID=168] [jobID=169] [query="CREATE TABLE IF NOT EXISTS `finishmark17000` (`foo` BIGINT PRIMARY KEY)"] [finishedTs=457152665072435213] [2025/04/06 08:23:43.559 +08:00] [INFO] [case.go:581] ["ddl synced"] [table=finishmark17000] [2025/04/06 08:23:43.559 +08:00] [INFO] [case.go:587] ["downstream sync success"] [endTs=457152665072435213] [2025/04/06 08:23:43.562 +08:00] [INFO] [case.go:191] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2025/04/06 08:23:43.625 +08:00] [INFO] [case.go:191] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2025/04/06 08:23:43.638 +08:00] [INFO] [case.go:308] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2025/04/06 08:23:43.730 +08:00] [INFO] [case.go:308] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] table force_replicate_table.t5 exists check diff failed 2-th time, retry later table ddl_only_block_related_table.finish_mark not exists for 2-th check, retry later table force_replicate_table.t6 not exists for 1-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 06 Apr 2025 00:23:44 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7770e90b-3741-4241-bfba-a07a0f0f5b72 {"id":"7770e90b-3741-4241-bfba-a07a0f0f5b72","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899021} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087b209cd9 7770e90b-3741-4241-bfba-a07a0f0f5b72 /tidb/cdc/default/default/upstream/7489989219320878993 {"id":7489989219320878993,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7770e90b-3741-4241-bfba-a07a0f0f5b72 {"id":"7770e90b-3741-4241-bfba-a07a0f0f5b72","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899021} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087b209cd9 7770e90b-3741-4241-bfba-a07a0f0f5b72 /tidb/cdc/default/default/upstream/7489989219320878993 {"id":7489989219320878993,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 06 Apr 2025 00:23:44 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/e684662d-022f-4a79-9ef3-0a59dba96193 {UpstreamID:7489988628380945915 Namespace:default ID:e684662d-022f-4a79-9ef3-0a59dba96193 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-06 08:21:28.032207553 +0800 CST StartTs:457152628763131907 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc004278120 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457152630100066306} {CheckpointTs:457152665594363910 MinTableBarrierTs:457152665594363910 AdminJobType:noop} span: {tableID:112, startKey:7480000000000000ff705f720000000000fa, endKey:7480000000000000ff705f730000000000fa}, resolvedTs: 457152665607208961, checkpointTs: 457152665594363910, state: Replicating span: {tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}, resolvedTs: 457152665594363911, checkpointTs: 457152665594363910, state: Replicating span: {tableID:116, startKey:7480000000000000ff745f720000000000fa, endKey:7480000000000000ff745f730000000000fa}, resolvedTs: 457152665594363910, checkpointTs: 457152665594363910, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f827be15-ddf1-4e48-bbf3-ff67b0f120b6 {"id":"f827be15-ddf1-4e48-bbf3-ff67b0f120b6","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899021} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 2 /tidb/cdc/default/__cdc_meta__/owner/2231960879054b56 f827be15-ddf1-4e48-bbf3-ff67b0f120b6 /tidb/cdc/default/default/changefeed/info/e684662d-022f-4a79-9ef3-0a59dba96193 {"upstream-id":7489988628380945915,"namespace":"default","changefeed-id":"e684662d-022f-4a79-9ef3-0a59dba96193","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-06T08:21:28.032207553+08:00","start-ts":457152628763131907,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457152630100066306} /tidb/cdc/default/default/changefeed/status/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":457152665594363910,"min-table-barrier-ts":457152665594363910,"admin-job-type":0} /tidb/cdc/default/default/task/position/f827be15-ddf1-4e48-bbf3-ff67b0f120b6/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489988628380945915 {"id":7489988628380945915,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/e684662d-022f-4a79-9ef3-0a59dba96193 {UpstreamID:7489988628380945915 Namespace:default ID:e684662d-022f-4a79-9ef3-0a59dba96193 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-06 08:21:28.032207553 +0800 CST StartTs:457152628763131907 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc004278120 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457152630100066306} {CheckpointTs:457152665594363910 MinTableBarrierTs:457152665594363910 AdminJobType:noop} span: {tableID:112, startKey:7480000000000000ff705f720000000000fa, endKey:7480000000000000ff705f730000000000fa}, resolvedTs: 457152665607208961, checkpointTs: 457152665594363910, state: Replicating span: {tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}, resolvedTs: 457152665594363911, checkpointTs: 457152665594363910, state: Replicating span: {tableID:116, startKey:7480000000000000ff745f720000000000fa, endKey:7480000000000000ff745f730000000000fa}, resolvedTs: 457152665594363910, checkpointTs: 457152665594363910, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f827be15-ddf1-4e48-bbf3-ff67b0f120b6 {"id":"f827be15-ddf1-4e48-bbf3-ff67b0f120b6","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899021} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 2 /tidb/cdc/default/__cdc_meta__/owner/2231960879054b56 f827be15-ddf1-4e48-bbf3-ff67b0f120b6 /tidb/cdc/default/default/changefeed/info/e684662d-022f-4a79-9ef3-0a59dba96193 {"upstream-id":7489988628380945915,"namespace":"default","changefeed-id":"e684662d-022f-4a79-9ef3-0a59dba96193","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-06T08:21:28.032207553+08:00","start-ts":457152628763131907,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457152630100066306} /tidb/cdc/default/default/changefeed/status/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":457152665594363910,"min-table-barrier-ts":457152665594363910,"admin-job-type":0} /tidb/cdc/default/default/task/position/f827be15-ddf1-4e48-bbf3-ff67b0f120b6/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489988628380945915 {"id":7489988628380945915,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/e684662d-022f-4a79-9ef3-0a59dba96193 {UpstreamID:7489988628380945915 Namespace:default ID:e684662d-022f-4a79-9ef3-0a59dba96193 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-06 08:21:28.032207553 +0800 CST StartTs:457152628763131907 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc004278120 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457152630100066306} {CheckpointTs:457152665594363910 MinTableBarrierTs:457152665594363910 AdminJobType:noop} span: {tableID:112, startKey:7480000000000000ff705f720000000000fa, endKey:7480000000000000ff705f730000000000fa}, resolvedTs: 457152665607208961, checkpointTs: 457152665594363910, state: Replicating span: {tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}, resolvedTs: 457152665594363911, checkpointTs: 457152665594363910, state: Replicating span: {tableID:116, startKey:7480000000000000ff745f720000000000fa, endKey:7480000000000000ff745f730000000000fa}, resolvedTs: 457152665594363910, checkpointTs: 457152665594363910, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f827be15-ddf1-4e48-bbf3-ff67b0f120b6 {"id":"f827be15-ddf1-4e48-bbf3-ff67b0f120b6","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899021} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 2 /tidb/cdc/default/__cdc_meta__/owner/2231960879054b56 f827be15-ddf1-4e48-bbf3-ff67b0f120b6 /tidb/cdc/default/default/changefeed/info/e684662d-022f-4a79-9ef3-0a59dba96193 {"upstream-id":7489988628380945915,"namespace":"default","changefeed-id":"e684662d-022f-4a79-9ef3-0a59dba96193","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-06T08:21:28.032207553+08:00","start-ts":457152628763131907,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457152630100066306} /tidb/cdc/default/default/changefeed/status/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":457152665594363910,"min-table-barrier-ts":457152665594363910,"admin-job-type":0} /tidb/cdc/default/default/task/position/f827be15-ddf1-4e48-bbf3-ff67b0f120b6/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489988628380945915 {"id":7489988628380945915,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x cdc.test cli capture list 2>&1 | grep '"is-owner": true' + set +x go: downloading go1.23.4 (linux/amd64) check diff successfully + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7770e90b-3741-4241-bfba-a07a0f0f5b72 {"id":"7770e90b-3741-4241-bfba-a07a0f0f5b72","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899021} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087b209cd9 7770e90b-3741-4241-bfba-a07a0f0f5b72 /tidb/cdc/default/default/upstream/7489989219320878993 {"id":7489989219320878993,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x table test.table1 not exists for 1-th check, retry later wait process cdc.test exit for 1-th time... table processor_delay.t40 not exists for 2-th check, retry later wait process cdc.test exit for 2-th time... "is-owner": true, run task successfully table test.t_100 not exists for 8-th check, retry later table force_replicate_table.t6 exists check_data_subset force_replicate_table.t0 127.0.0.1 4000 127.0.0.1 3306 table test.finish_mark not exists for 22-th check, retry later wait process cdc.test exit for 3-th time... [2025/04/06 08:23:46.126 +08:00] [INFO] [case.go:544] ["upstream create table executed"] [tblName=finishmark18000] [2025/04/06 08:23:46.126 +08:00] [INFO] [case.go:576] ["downstream start wait for table"] [tblName=finishmark18000] \033[0;36m<<< Run all test success >>>\033[0m check diff failed 3-th time, retry later cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Sun Apr 6 08:23:46 CST 2025] <<<<<< run test case event_filter success! >>>>>> table ddl_only_block_related_table.finish_mark not exists for 3-th check, retry later [Pipeline] } [Pipeline] // timeout [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-periodics_cdc_integration_mysql_test-234/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // stage [Pipeline] } run task successfully check_data_subset force_replicate_table.t1 127.0.0.1 4000 127.0.0.1 3306 [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } owner pid: 4905 owner id f827be15-ddf1-4e48-bbf3-ff67b0f120b6 default owner key /tidb/cdc/default/__cdc_meta__/owner/2231960879054b56 [Sun Apr 6 08:23:47 CST 2025] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + GO_FAILPOINTS= + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.50225024.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_gap_between_watch_capture.server2.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_gap_between_watch_capture.server2 --cluster-id default --addr 127.0.0.1:8301 ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [Pipeline] // node table test.table1 exists table test.table2 not exists for 1-th check, retry later [Pipeline] } [Pipeline] // podTemplate [Pipeline] } run task successfully check_data_subset force_replicate_table.t2 127.0.0.1 4000 127.0.0.1 3306 [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } table test.t_100 not exists for 9-th check, retry later table processor_delay.t40 exists table processor_delay.t41 not exists for 1-th check, retry later table test.finish_mark not exists for 23-th check, retry later [2025/04/06 08:23:48.126 +08:00] [INFO] [case.go:687] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark18000] [2025/04/06 08:23:48.134 +08:00] [INFO] [case.go:697] ["try parse finishedTs from ticdc log"] [tableName=finishmark18000] /tmp/tidb_cdc_test/bank/cdcdown.log [2025/04/06 08:23:48.136 +08:00] [INFO] [case.go:717] ["total files"] [file="[/tmp/tidb_cdc_test/bank/cdcdown.log]"] check diff failed 4-th time, retry later table ddl_only_block_related_table.finish_mark exists run task successfully check_data_subset force_replicate_table.t3 127.0.0.1 4000 127.0.0.1 3306 wait process 8062 exit for 1-th time... run task successfully check_data_subset force_replicate_table.t4 127.0.0.1 4000 127.0.0.1 3306 table test.table2 exists check diff failed 1-th time, retry later wait process 8062 exit for 2-th time... table test.t_100 exists /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (8062) - No such process wait process 8062 exit for 3-th time... process 8062 already exit [Sun Apr 6 08:23:49 CST 2025] <<<<<< START cdc server in ddl_only_block_related_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ExecuteNotDone=return(true)' + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_only_block_related_table.81928194.out server --log-file /tmp/tidb_cdc_test/ddl_only_block_related_table/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_only_block_related_table/cdc_data --cluster-id default + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table processor_delay.t41 exists table processor_delay.t42 not exists for 1-th check, retry later table test.finish_mark not exists for 24-th check, retry later run task successfully check_data_subset force_replicate_table.t5 127.0.0.1 4000 127.0.0.1 3306 table batch_add_table.finish_mark not exists for 1-th check, retry later found first match line: 457152666697728014 : [2025/04/06 08:23:48.626 +08:00] [INFO] [ddl_manager.go:298] ["handle a ddl job"] [namespace=default] [changefeed=changefeed-for-find-finished-ts] [tableID=170] [jobID=171] [query="CREATE TABLE IF NOT EXISTS `finishmark18000` (`foo` BIGINT PRIMARY KEY)"] [finishedTs=457152666697728014] [2025/04/06 08:23:49.641 +08:00] [INFO] [case.go:581] ["ddl synced"] [table=finishmark18000] [2025/04/06 08:23:49.641 +08:00] [INFO] [case.go:587] ["downstream sync success"] [endTs=457152666697728014] [2025/04/06 08:23:49.643 +08:00] [INFO] [case.go:191] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2025/04/06 08:23:49.646 +08:00] [INFO] [case.go:191] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2025/04/06 08:23:49.727 +08:00] [INFO] [case.go:308] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2025/04/06 08:23:49.736 +08:00] [INFO] [case.go:308] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] check diff failed 5-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 06 Apr 2025 00:23:50 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/e684662d-022f-4a79-9ef3-0a59dba96193 {UpstreamID:7489988628380945915 Namespace:default ID:e684662d-022f-4a79-9ef3-0a59dba96193 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-06 08:21:28.032207553 +0800 CST StartTs:457152628763131907 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0044205a0 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457152630100066306} {CheckpointTs:457152667180072961 MinTableBarrierTs:457152667180072962 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/73b3e0ab-69b9-491b-9339-2f87b0365417 {"id":"73b3e0ab-69b9-491b-9339-2f87b0365417","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899027} /tidb/cdc/default/__cdc_meta__/capture/f827be15-ddf1-4e48-bbf3-ff67b0f120b6 {"id":"f827be15-ddf1-4e48-bbf3-ff67b0f120b6","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899021} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 2 /tidb/cdc/default/__cdc_meta__/owner/2231960879054b56 f827be15-ddf1-4e48-bbf3-ff67b0f120b6 /tidb/cdc/default/__cdc_meta__/owner/2231960879054ba9 73b3e0ab-69b9-491b-9339-2f87b0365417 /tidb/cdc/default/default/changefeed/info/e684662d-022f-4a79-9ef3-0a59dba96193 {"upstream-id":7489988628380945915,"namespace":"default","changefeed-id":"e684662d-022f-4a79-9ef3-0a59dba96193","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-06T08:21:28.032207553+08:00","start-ts":457152628763131907,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457152630100066306} /tidb/cdc/default/default/changefeed/status/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":457152667180072961,"min-table-barrier-ts":457152667180072962,"admin-job-type":0} /tidb/cdc/default/default/task/position/73b3e0ab-69b9-491b-9339-2f87b0365417/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/f827be15-ddf1-4e48-bbf3-ff67b0f120b6/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489988628380945915 {"id":7489988628380945915,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/e684662d-022f-4a79-9ef3-0a59dba96193 {UpstreamID:7489988628380945915 Namespace:default ID:e684662d-022f-4a79-9ef3-0a59dba96193 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-06 08:21:28.032207553 +0800 CST StartTs:457152628763131907 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0044205a0 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457152630100066306} {CheckpointTs:457152667180072961 MinTableBarrierTs:457152667180072962 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/73b3e0ab-69b9-491b-9339-2f87b0365417 {"id":"73b3e0ab-69b9-491b-9339-2f87b0365417","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899027} /tidb/cdc/default/__cdc_meta__/capture/f827be15-ddf1-4e48-bbf3-ff67b0f120b6 {"id":"f827be15-ddf1-4e48-bbf3-ff67b0f120b6","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899021} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 2 /tidb/cdc/default/__cdc_meta__/owner/2231960879054b56 f827be15-ddf1-4e48-bbf3-ff67b0f120b6 /tidb/cdc/default/__cdc_meta__/owner/2231960879054ba9 73b3e0ab-69b9-491b-9339-2f87b0365417 /tidb/cdc/default/default/changefeed/info/e684662d-022f-4a79-9ef3-0a59dba96193 {"upstream-id":7489988628380945915,"namespace":"default","changefeed-id":"e684662d-022f-4a79-9ef3-0a59dba96193","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-06T08:21:28.032207553+08:00","start-ts":457152628763131907,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457152630100066306} /tidb/cdc/default/default/changefeed/status/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":457152667180072961,"min-table-barrier-ts":457152667180072962,"admin-job-type":0} /tidb/cdc/default/default/task/position/73b3e0ab-69b9-491b-9339-2f87b0365417/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/f827be15-ddf1-4e48-bbf3-ff67b0f120b6/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489988628380945915 {"id":7489988628380945915,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: changefeedID: default/e684662d-022f-4a79-9ef3-0a59dba96193 {UpstreamID:7489988628380945915 Namespace:default ID:e684662d-022f-4a79-9ef3-0a59dba96193 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-06 08:21:28.032207553 +0800 CST StartTs:457152628763131907 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0044205a0 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457152630100066306} {CheckpointTs:457152667180072961 MinTableBarrierTs:457152667180072962 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/73b3e0ab-69b9-491b-9339-2f87b0365417 {"id":"73b3e0ab-69b9-491b-9339-2f87b0365417","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899027} /tidb/cdc/default/__cdc_meta__/capture/f827be15-ddf1-4e48-bbf3-ff67b0f120b6 {"id":"f827be15-ddf1-4e48-bbf3-ff67b0f120b6","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899021} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 2 /tidb/cdc/default/__cdc_meta__/owner/2231960879054b56 f827be15-ddf1-4e48-bbf3-ff67b0f120b6 /tidb/cdc/default/__cdc_meta__/owner/2231960879054ba9 73b3e0ab-69b9-491b-9339-2f87b0365417 /tidb/cdc/default/default/changefeed/info/e684662d-022f-4a79-9ef3-0a59dba96193 {"upstream-id":7489988628380945915,"namespace":"default","changefeed-id":"e684662d-022f-4a79-9ef3-0a59dba96193","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-06T08:21:28.032207553+08:00","start-ts":457152628763131907,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457152630100066306} /tidb/cdc/default/default/changefeed/status/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":457152667180072961,"min-table-barrier-ts":457152667180072962,"admin-job-type":0} /tidb/cdc/default/default/task/position/73b3e0ab-69b9-491b-9339-2f87b0365417/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/f827be15-ddf1-4e48-bbf3-ff67b0f120b6/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489988628380945915 {"id":7489988628380945915,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x cdc.test cli capture list 2>&1 | grep -v "f827be15-ddf1-4e48-bbf3-ff67b0f120b6 default" | grep id [2025/04/06 08:23:50.633 +08:00] [INFO] [case.go:544] ["upstream create table executed"] [tblName=finishmark19000] [2025/04/06 08:23:50.633 +08:00] [INFO] [case.go:576] ["downstream start wait for table"] [tblName=finishmark19000] run task successfully check_data_subset force_replicate_table.t6 127.0.0.1 4000 127.0.0.1 3306 id=19,a=NULL doesn't exist in downstream table force_replicate_table.t6 run task failed 1-th time, retry later go: downloading github.com/pingcap/errors v0.11.5-0.20240318064555-6bd07397691f go: downloading github.com/pingcap/log v1.1.1-0.20241212030209-7e3ff8601a2a go: downloading go.uber.org/zap v1.27.0 go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20241223052309-3735ed55a394 go: downloading github.com/BurntSushi/toml v1.4.1-0.20240526193622-a339e1f7089c go: downloading github.com/pingcap/failpoint v0.0.0-20240528011301-b51a646c7c86 go: downloading gopkg.in/natefinch/lumberjack.v2 v2.2.1 go: downloading go.uber.org/atomic v1.11.0 go: downloading go.uber.org/multierr v1.11.0 check diff successfully table processor_delay.t42 exists table processor_delay.t43 not exists for 1-th check, retry later "id": "73b3e0ab-69b9-491b-9339-2f87b0365417", run task successfully table test.finish_mark not exists for 25-th check, retry later table batch_add_table.finish_mark not exists for 2-th check, retry later Resume changefeed with checkpointTs3 457152667736342530 table test.table5 not exists for 1-th check, retry later check diff successfully check_data_subset force_replicate_table.t6 127.0.0.1 4000 127.0.0.1 3306 go: downloading github.com/go-sql-driver/mysql v1.7.1 go: downloading github.com/pingcap/tidb/pkg/parser v0.0.0-20241223052309-3735ed55a394 go: downloading github.com/tikv/pd/client v0.0.0-20241111073742-238d4d79ea31 go: downloading github.com/pingcap/kvproto v0.0.0-20241120022153-92b0414aeed8 go: downloading github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 go: downloading github.com/json-iterator/go v1.1.12 go: downloading github.com/prometheus/client_golang v1.20.5 go: downloading github.com/opentracing/opentracing-go v1.2.0 go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 go: downloading github.com/golang/protobuf v1.5.4 go: downloading github.com/influxdata/tdigest v0.0.1 go: downloading github.com/danjacques/gofslock v0.0.0-20240212154529-d899e02bfe22 go: downloading github.com/pingcap/tipb v0.0.0-20241105053214-f91fdb81a69e go: downloading golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 go: downloading github.com/coocood/freecache v1.2.1 go: downloading github.com/spf13/pflag v1.0.5 go: downloading github.com/tikv/client-go/v2 v2.0.8-0.20241209094930-06d7f4b9233b go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible go: downloading golang.org/x/sync v0.10.0 go: downloading github.com/opentracing/basictracer-go v1.1.0 go: downloading github.com/prometheus/client_model v0.6.1 go: downloading golang.org/x/tools v0.28.0 go: downloading github.com/gorilla/mux v1.8.1 go: downloading gopkg.in/yaml.v2 v2.4.0 go: downloading github.com/coreos/go-semver v0.3.1 go: downloading github.com/pingcap/sysutil v1.0.1-0.20240311050922-ae81ee01f3a5 go: downloading github.com/docker/go-units v0.5.0 go: downloading github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a go: downloading github.com/bits-and-blooms/bitset v1.14.3 go: downloading golang.org/x/net v0.33.0 go: downloading golang.org/x/text v0.21.0 go: downloading github.com/stretchr/testify v1.10.0 go: downloading go.etcd.io/etcd/client/v3 v3.5.12 go: downloading github.com/google/uuid v1.6.0 go: downloading github.com/shirou/gopsutil/v3 v3.24.5 go: downloading google.golang.org/grpc v1.64.0 go: downloading github.com/google/btree v1.1.2 go: downloading github.com/otiai10/copy v1.2.0 go: downloading github.com/tidwall/btree v1.7.0 go: downloading github.com/twmb/murmur3 v1.1.6 go: downloading golang.org/x/sys v0.28.0 go: downloading cloud.google.com/go/storage v1.39.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v1.16.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 go: downloading github.com/aws/aws-sdk-go v1.55.5 go: downloading github.com/tikv/pd v1.1.0-beta.0.20240407022249-7179657d129b go: downloading github.com/go-resty/resty/v2 v2.11.0 go: downloading github.com/klauspost/compress v1.17.9 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 06 Apr 2025 00:23:52 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/ddl-only-block-related-table {UpstreamID:7489989193297080848 Namespace:default ID:ddl-only-block-related-table SinkURI:mysql://root@127.0.0.1:3306/ CreateTime:2025-04-06 08:23:37.983731343 +0800 CST StartTs:457152664145494017 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc004389200 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457152664158601224} {CheckpointTs:457152667867676676 MinTableBarrierTs:457152667867676676 AdminJobType:noop} span: {tableID:118, startKey:7480000000000000ff765f720000000000fa, endKey:7480000000000000ff765f730000000000fa}, resolvedTs: 457152667867676676, checkpointTs: 457152667867676676, state: Replicating span: {tableID:120, startKey:7480000000000000ff785f720000000000fa, endKey:7480000000000000ff785f730000000000fa}, resolvedTs: 457152667867676676, checkpointTs: 457152667867676676, state: Replicating span: {tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}, resolvedTs: 457152667867676676, checkpointTs: 457152667867676676, state: Replicating span: {tableID:116, startKey:7480000000000000ff745f720000000000fa, endKey:7480000000000000ff745f730000000000fa}, resolvedTs: 457152667880783878, checkpointTs: 457152667867676676, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ee2da332-df96-43ac-9dfb-384b1a6c15cb {"id":"ee2da332-df96-43ac-9dfb-384b1a6c15cb","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899029} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087b0810aa ee2da332-df96-43ac-9dfb-384b1a6c15cb /tidb/cdc/default/default/changefeed/info/ddl-only-block-related-table {"upstream-id":7489989193297080848,"namespace":"default","changefeed-id":"ddl-only-block-related-table","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2025-04-06T08:23:37.983731343+08:00","start-ts":457152664145494017,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457152664158601224} /tidb/cdc/default/default/changefeed/status/ddl-only-block-related-table {"checkpoint-ts":457152667867676676,"min-table-barrier-ts":457152667867676676,"admin-job-type":0} /tidb/cdc/default/default/task/position/ee2da332-df96-43ac-9dfb-384b1a6c15cb/ddl-only-block-related-table {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489989193297080848 {"id":7489989193297080848,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-only-block-related-table {UpstreamID:7489989193297080848 Namespace:default ID:ddl-only-block-related-table SinkURI:mysql://root@127.0.0.1:3306/ CreateTime:2025-04-06 08:23:37.983731343 +0800 CST StartTs:457152664145494017 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc004389200 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457152664158601224} {CheckpointTs:457152667867676676 MinTableBarrierTs:457152667867676676 AdminJobType:noop} span: {tableID:118, startKey:7480000000000000ff765f720000000000fa, endKey:7480000000000000ff765f730000000000fa}, resolvedTs: 457152667867676676, checkpointTs: 457152667867676676, state: Replicating span: {tableID:120, startKey:7480000000000000ff785f720000000000fa, endKey:7480000000000000ff785f730000000000fa}, resolvedTs: 457152667867676676, checkpointTs: 457152667867676676, state: Replicating span: {tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}, resolvedTs: 457152667867676676, checkpointTs: 457152667867676676, state: Replicating span: {tableID:116, startKey:7480000000000000ff745f720000000000fa, endKey:7480000000000000ff745f730000000000fa}, resolvedTs: 457152667880783878, checkpointTs: 457152667867676676, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ee2da332-df96-43ac-9dfb-384b1a6c15cb {"id":"ee2da332-df96-43ac-9dfb-384b1a6c15cb","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899029} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087b0810aa ee2da332-df96-43ac-9dfb-384b1a6c15cb /tidb/cdc/default/default/changefeed/info/ddl-only-block-related-table {"upstream-id":7489989193297080848,"namespace":"default","changefeed-id":"ddl-only-block-related-table","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2025-04-06T08:23:37.983731343+08:00","start-ts":457152664145494017,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457152664158601224} /tidb/cdc/default/default/changefeed/status/ddl-only-block-related-table {"checkpoint-ts":457152667867676676,"min-table-barrier-ts":457152667867676676,"admin-job-type":0} /tidb/cdc/default/default/task/position/ee2da332-df96-43ac-9dfb-384b1a6c15cb/ddl-only-block-related-table {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489989193297080848 {"id":7489989193297080848,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-only-block-related-table {UpstreamID:7489989193297080848 Namespace:default ID:ddl-only-block-related-table SinkURI:mysql://root@127.0.0.1:3306/ CreateTime:2025-04-06 08:23:37.983731343 +0800 CST StartTs:457152664145494017 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc004389200 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457152664158601224} {CheckpointTs:457152667867676676 MinTableBarrierTs:457152667867676676 AdminJobType:noop} span: {tableID:118, startKey:7480000000000000ff765f720000000000fa, endKey:7480000000000000ff765f730000000000fa}, resolvedTs: 457152667867676676, checkpointTs: 457152667867676676, state: Replicating span: {tableID:120, startKey:7480000000000000ff785f720000000000fa, endKey:7480000000000000ff785f730000000000fa}, resolvedTs: 457152667867676676, checkpointTs: 457152667867676676, state: Replicating span: {tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}, resolvedTs: 457152667867676676, checkpointTs: 457152667867676676, state: Replicating span: {tableID:116, startKey:7480000000000000ff745f720000000000fa, endKey:7480000000000000ff745f730000000000fa}, resolvedTs: 457152667880783878, checkpointTs: 457152667867676676, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ee2da332-df96-43ac-9dfb-384b1a6c15cb {"id":"ee2da332-df96-43ac-9dfb-384b1a6c15cb","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899029} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087b0810aa ee2da332-df96-43ac-9dfb-384b1a6c15cb /tidb/cdc/default/default/changefeed/info/ddl-only-block-related-table {"upstream-id":7489989193297080848,"namespace":"default","changefeed-id":"ddl-only-block-related-table","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2025-04-06T08:23:37.983731343+08:00","start-ts":457152664145494017,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457152664158601224} /tidb/cdc/default/default/changefeed/status/ddl-only-block-related-table {"checkpoint-ts":457152667867676676,"min-table-barrier-ts":457152667867676676,"admin-job-type":0} /tidb/cdc/default/default/task/position/ee2da332-df96-43ac-9dfb-384b1a6c15cb/ddl-only-block-related-table {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489989193297080848 {"id":7489989193297080848,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x go: downloading github.com/ks3sdklib/aws-sdk-go v1.2.9 go: downloading cloud.google.com/go v0.112.2 go: downloading golang.org/x/oauth2 v0.24.0 go: downloading google.golang.org/api v0.170.0 [2025/04/06 08:23:52.633 +08:00] [INFO] [case.go:687] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark19000] go: downloading github.com/pkg/errors v0.9.1 go: downloading github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef go: downloading github.com/yangkeao/ldap/v3 v3.4.5-0.20230421065457-369a3bab1117 go: downloading github.com/gogo/protobuf v1.3.2 go: downloading github.com/qri-io/jsonschema v0.2.1 go: downloading google.golang.org/protobuf v1.35.2 go: downloading github.com/dolthub/swiss v0.2.1 go: downloading go.etcd.io/etcd/api/v3 v3.5.12 go: downloading github.com/cespare/xxhash/v2 v2.3.0 go: downloading go.uber.org/mock v0.4.0 go: downloading github.com/cockroachdb/pebble v1.1.0 go: downloading golang.org/x/time v0.7.0 go: downloading github.com/carlmjohnson/flagext v0.21.0 go: downloading github.com/jfcg/sorty/v2 v2.1.0 go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.12 go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 go: downloading github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 go: downloading golang.org/x/crypto v0.31.0 go: downloading github.com/beorn7/perks v1.0.1 go: downloading github.com/prometheus/common v0.61.0 go: downloading github.com/prometheus/procfs v0.15.1 go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible table processor_delay.t43 exists table processor_delay.t44 not exists for 1-th check, retry later check diff failed 1-th time, retry later check diff failed 1-th time, retry later run task successfully capture_id: 73b3e0ab-69b9-491b-9339-2f87b0365417 1 ETCDCTL_API=3 etcdctl get /tidb/cdc/default/__cdc_meta__/owner --prefix | grep '73b3e0ab-69b9-491b-9339-2f87b0365417' 73b3e0ab-69b9-491b-9339-2f87b0365417 run task successfully curl -X GET http://127.0.0.1:8300/status | grep '"is_owner": false' % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 182 100 182 0 0 15846 0 --:--:-- --:--:-- --:--:-- 16545 "is_owner": false run task successfully go: downloading github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 go: downloading github.com/golang/snappy v0.0.4 go: downloading github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc go: downloading github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 go: downloading github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df go: downloading github.com/dgraph-io/ristretto v0.1.1 go: downloading github.com/jellydator/ttlcache/v3 v3.0.1 go: downloading github.com/lestrrat-go/jwx/v2 v2.0.21 go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading github.com/dgryski/go-farm v0.0.0-20240924180020-3414d57e47da go: downloading github.com/tklauser/go-sysconf v0.3.12 go: downloading github.com/cloudfoundry/gosigar v1.3.6 go: downloading github.com/xitongsys/parquet-go v1.6.3-0.20240520233950-75e935fc3e17 go: downloading github.com/spkg/bom v1.0.0 go: downloading github.com/jedib0t/go-pretty/v6 v6.2.2 go: downloading cloud.google.com/go/compute/metadata v0.3.0 table test.finish_mark not exists for 26-th check, retry later go: downloading github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 go: downloading github.com/go-asn1-ber/asn1-ber v1.5.4 # diff Configuration. check-thread-count = 4 export-fix-sql = true check-struct-only = false [task] output-dir = "/tmp/tidb_cdc_test/consistent_replicate_nfs/sync_diff/output" source-instances = ["mysql1"] target-instance = "tidb0" target-check-tables = ["consistent_replicate_nfs.usertable*"] [data-sources] [data-sources.mysql1] host = "127.0.0.1" port = 4000 user = "root" password = "" snapshot = "457152663737335818" [data-sources.tidb0] host = "127.0.0.1" port = 3306 user = "root" password = "" [2025/04/06 08:23:51.839 +08:00] [INFO] [helper.go:54] ["init log"] [file=] [level=info] [2025/04/06 08:23:51.841 +08:00] [INFO] [apply.go:78] ["set memory limit"] [memoryLimit=10737418240] [2025/04/06 08:23:51.841 +08:00] [INFO] [apply.go:94] ["Start http pprof server"] [addr=127.0.0.1:6060] [2025/04/06 08:23:51.841 +08:00] [INFO] [mem_quota.go:82] ["New memory quota"] [namespace=default] [changefeed=redo-applier] [total=1073741824] [2025/04/06 08:23:51.841 +08:00] [INFO] [redo.go:167] ["apply redo log starts"] [checkpointTs=457152663527620742] [resolvedTs=457152663737335818] [2025/04/06 08:23:51.841 +08:00] [INFO] [file.go:114] ["succeed to download and sort redo logs"] [type=ddl] [duration=347.301µs] [2025/04/06 08:23:51.842 +08:00] [INFO] [tz.go:35] ["Use the timezone of the TiCDC server machine"] [timezoneName=System] [timezone=Asia/Shanghai] [2025/04/06 08:23:51.842 +08:00] [WARN] [config.go:428] ["Because time-zone is not specified, the timezone of the TiCDC server will be used. We recommend that you specify the time-zone explicitly. Please make sure that the timezone of the TiCDC server, sink-uri and the downstream database are consistent. If the downstream database does not load the timezone information, you can refer to https://dev.mysql.com/doc/refman/8.0/en/mysql-tzinfo-to-sql.html."] [timezone=Asia/Shanghai] [2025/04/06 08:23:51.842 +08:00] [ERROR] [config.go:216] ["The TiDB source ID should never be set to 0. Please report it as a bug. The default value will be used: 1."] [tidbSourceID=0] [2025/04/06 08:23:51.847 +08:00] [INFO] [db_helper.go:161] ["sink uri is configured"] [dsn="normal:******@tcp(127.0.0.1:3306)/?interpolateParams=true&multiStatements=true&allow_auto_random_explicit_insert=1&charset=utf8mb4&foreign_key_checks=0&maxAllowedPacket=0&readTimeout=2m&sql_mode=%22NO_AUTO_VALUE_ON_ZERO%2CNO_ENGINE_SUBSTITUTION%2CONLY_FULL_GROUP_BY%2CALLOW_INVALID_DATES%2CIGNORE_SPACE%22&tidb_enable_external_ts_read=%22OFF%22&tidb_placement_mode=%22ignore%22&tidb_txn_mode=optimistic&time_zone=%22Asia%2FShanghai%22&timeout=2m&transaction_isolation=%22READ-COMMITTED%22&writeTimeout=2m"] [2025/04/06 08:23:51.851 +08:00] [INFO] [mysql.go:195] ["MySQL backends is created"] [changefeed=.] [workerCount=16] [forceReplicate=false] [2025/04/06 08:23:51.855 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/06 08:23:51.855 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/06 08:23:51.855 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/06 08:23:51.855 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/06 08:23:51.855 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/06 08:23:51.855 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/06 08:23:51.855 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/06 08:23:51.855 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/06 08:23:51.855 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/06 08:23:51.855 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/06 08:23:51.855 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/06 08:23:51.855 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/06 08:23:51.855 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/06 08:23:51.855 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/06 08:23:51.855 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/06 08:23:51.855 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/06 08:23:51.855 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=2] [2025/04/06 08:23:51.855 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=1] [2025/04/06 08:23:51.855 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=5] [2025/04/06 08:23:51.855 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=9] [2025/04/06 08:23:51.855 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=7] [2025/04/06 08:23:51.855 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=4] [2025/04/06 08:23:51.855 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=3] [2025/04/06 08:23:51.855 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=10] [2025/04/06 08:23:51.855 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=12] [2025/04/06 08:23:51.855 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=13] [2025/04/06 08:23:51.855 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=15] [2025/04/06 08:23:51.855 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=6] [2025/04/06 08:23:51.855 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=0] [2025/04/06 08:23:51.855 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=8] [2025/04/06 08:23:51.855 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=11] [2025/04/06 08:23:51.855 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=14] [2025/04/06 08:23:51.855 +08:00] [INFO] [tz.go:35] ["Use the timezone of the TiCDC server machine"] [timezoneName=System] [timezone=Asia/Shanghai] [2025/04/06 08:23:51.855 +08:00] [WARN] [config.go:428] ["Because time-zone is not specified, the timezone of the TiCDC server will be used. We recommend that you specify the time-zone explicitly. Please make sure that the timezone of the TiCDC server, sink-uri and the downstream database are consistent. If the downstream database does not load the timezone information, you can refer to https://dev.mysql.com/doc/refman/8.0/en/mysql-tzinfo-to-sql.html."] [timezone=Asia/Shanghai] [2025/04/06 08:23:51.855 +08:00] [ERROR] [config.go:216] ["The TiDB source ID should never be set to 0. Please report it as a bug. The default value will be used: 1."] [tidbSourceID=0] [2025/04/06 08:23:51.858 +08:00] [INFO] [file.go:284] ["ignore logs which commitTs is greater than resolvedTs"] [filename=127.0.0.1:8300_d94e038a-640d-41d9-8b2a-c5dbe380d0f7_row_457152664340267154_2ac21eda-cbbb-44f9-b4d6-1ecf61066103.log] [endTs=457152663737335818] [2025/04/06 08:23:51.859 +08:00] [INFO] [file.go:284] ["ignore logs which commitTs is greater than resolvedTs"] [filename=127.0.0.1:8300_d94e038a-640d-41d9-8b2a-c5dbe380d0f7_row_457152663737335831_ed7dd70c-ab89-4c8a-ba40-8958ba8d0085.log] [endTs=457152663737335818] [2025/04/06 08:23:51.859 +08:00] [INFO] [db_helper.go:161] ["sink uri is configured"] [dsn="normal:******@tcp(127.0.0.1:3306)/?interpolateParams=true&multiStatements=true&allow_auto_random_explicit_insert=1&charset=utf8mb4&foreign_key_checks=0&maxAllowedPacket=0&readTimeout=2m&sql_mode=%22NO_AUTO_VALUE_ON_ZERO%2CNO_ENGINE_SUBSTITUTION%2CONLY_FULL_GROUP_BY%2CALLOW_INVALID_DATES%2CIGNORE_SPACE%22&tidb_enable_external_ts_read=%22OFF%22&tidb_placement_mode=%22ignore%22&tidb_txn_mode=optimistic&time_zone=%22Asia%2FShanghai%22&timeout=2m&transaction_isolation=%22READ-COMMITTED%22&writeTimeout=2m"] [2025/04/06 08:23:51.860 +08:00] [INFO] [file.go:114] ["succeed to download and sort redo logs"] [type=row] [duration=18.930584ms] [2025/04/06 08:23:51.862 +08:00] [WARN] [mysql_ddl_sink.go:291] ["please set `has-vector-type` to be true if a column is vector type when the downstream is not TiDB or TiDB version less than specify version"] [hasVectorType=false] [supportVectorVersion=8.4.0] [2025/04/06 08:23:51.862 +08:00] [INFO] [mysql_ddl_sink.go:122] ["MySQL DDL sink is created"] [namespace=] [changefeed=] [2025/04/06 08:23:52.072 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [span="{tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}"] [checkpointTs=457152663737335818] [2025/04/06 08:23:52.073 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [span="{tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}"] [checkpointTs=457152663737335818] [2025/04/06 08:23:52.173 +08:00] [INFO] [redo.go:223] ["apply redo log finishes"] [appliedLogCount=1181] [appliedDDLCount=0] [currentCheckpoint=457152663737335818] [2025/04/06 08:23:52.173 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=6] [2025/04/06 08:23:52.173 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=0] [2025/04/06 08:23:52.173 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=9] [2025/04/06 08:23:52.173 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=7] [2025/04/06 08:23:52.173 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=14] [2025/04/06 08:23:52.173 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=5] [2025/04/06 08:23:52.173 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=12] [2025/04/06 08:23:52.173 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=1] [2025/04/06 08:23:52.173 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=11] [2025/04/06 08:23:52.173 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=13] [2025/04/06 08:23:52.173 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=4] [2025/04/06 08:23:52.173 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=10] [2025/04/06 08:23:52.173 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=2] [2025/04/06 08:23:52.173 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=15] [2025/04/06 08:23:52.173 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=3] [2025/04/06 08:23:52.173 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=8] Apply redo log successfully check diff successfully [Sun Apr 6 08:23:52 CST 2025] <<<<<< run test case consistent_replicate_nfs success! >>>>>> count(*) 46020 table test.finish_mark not exists for 1-th check, retry later table test.finish_mark not exists for 2-th check, retry later table test.finish_mark not exists for 3-th check, retry later table test.finish_mark exists check diff successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Sun Apr 6 08:23:53 CST 2025] <<<<<< run test case many_pk_or_uk success! >>>>>> table test.table5 not exists for 2-th check, retry later go: downloading github.com/dolthub/maphash v0.1.0 go: downloading github.com/jfcg/sixb v1.3.8 go: downloading google.golang.org/genproto/googleapis/api v0.0.0-20240401170217-c3f982113cda go: downloading cloud.google.com/go/iam v1.1.7 wait process cdc.test exit for 1-th time... table batch_add_table.finish_mark not exists for 3-th check, retry later go: downloading github.com/googleapis/gax-go/v2 v2.12.3 go: downloading google.golang.org/genproto v0.0.0-20240401170217-c3f982113cda go: downloading github.com/pingcap/badger v1.5.1-0.20241015064302-38533b6cbf8d go: downloading github.com/google/pprof v0.0.0-20241001023024-f4c0cfd0cf1d go: downloading github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f go: downloading github.com/coreos/go-systemd/v22 v22.5.0 go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading github.com/cheggaaa/pb/v3 v3.0.8 go: downloading golang.org/x/term v0.27.0 go: downloading github.com/qri-io/jsonpointer v0.1.1 go: downloading github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c wait process cdc.test exit for 2-th time... go: downloading github.com/robfig/cron/v3 v3.0.1 go: downloading github.com/kylelemons/godebug v1.1.0 go: downloading github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 go: downloading github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec go: downloading github.com/tklauser/numcpus v0.6.1 go: downloading github.com/apache/thrift v0.16.0 go: downloading github.com/mattn/go-runewidth v0.0.16 go: downloading github.com/lestrrat-go/blackmagic v1.0.2 go: downloading github.com/robfig/cron v1.2.0 go: downloading github.com/lestrrat-go/httprc v1.0.5 go: downloading github.com/lestrrat-go/iter v1.0.2 go: downloading github.com/lestrrat-go/option v1.0.1 go: downloading go.opencensus.io v0.23.1-0.20220331163232-052120675fac go: downloading go.opentelemetry.io/otel v1.24.0 go: downloading go.opentelemetry.io/otel/trace v1.24.0 go: downloading google.golang.org/genproto/googleapis/rpc v0.0.0-20240515191416-fc5f0ca64291 go: downloading github.com/golang-jwt/jwt/v5 v5.2.1 go: downloading github.com/golang/glog v1.2.0 go: downloading github.com/dustin/go-humanize v1.0.1 go: downloading github.com/VividCortex/ewma v1.2.0 go: downloading github.com/fatih/color v1.18.0 go: downloading github.com/mattn/go-colorable v0.1.13 go: downloading github.com/mattn/go-isatty v0.0.20 go: downloading github.com/apache/arrow/go/v12 v12.0.1 go: downloading github.com/goccy/go-reflect v1.2.0 go: downloading github.com/rivo/uniseg v0.4.7 go: downloading github.com/pierrec/lz4/v4 v4.1.18 go: downloading github.com/lestrrat-go/httpcc v1.0.1 go: downloading github.com/golang-jwt/jwt v3.2.2+incompatible go: downloading github.com/ncw/directio v1.0.5 go: downloading github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 go: downloading github.com/klauspost/cpuid v1.3.1 go: downloading github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Sun Apr 6 08:23:54 CST 2025] <<<<<< run test case force_replicate_table success! >>>>>> [2025/04/06 08:23:54.644 +08:00] [INFO] [case.go:687] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark19000] [2025/04/06 08:23:54.735 +08:00] [INFO] [case.go:697] ["try parse finishedTs from ticdc log"] [tableName=finishmark19000] /tmp/tidb_cdc_test/bank/cdcdown.log [2025/04/06 08:23:54.737 +08:00] [INFO] [case.go:717] ["total files"] [file="[/tmp/tidb_cdc_test/bank/cdcdown.log]"] go: downloading github.com/go-logr/logr v1.4.1 go: downloading github.com/go-logr/stdr v1.2.2 go: downloading go.opentelemetry.io/otel/metric v1.24.0 table processor_delay.t44 exists table processor_delay.t45 exists table processor_delay.t46 not exists for 1-th check, retry later go: downloading github.com/cockroachdb/errors v1.11.1 go: downloading github.com/cockroachdb/redact v1.1.5 go: downloading github.com/DataDog/zstd v1.5.5 go: downloading github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 check diff failed 2-th time, retry later go: downloading github.com/getsentry/sentry-go v0.27.0 go: downloading github.com/kr/pretty v0.3.1 go: downloading github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b go: downloading github.com/rogpeppe/go-internal v1.13.1 go: downloading github.com/kr/text v0.2.0 check diff failed 2-th time, retry later table test.finish_mark not exists for 27-th check, retry later table batch_add_table.finish_mark not exists for 4-th check, retry later go: downloading golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 go: downloading github.com/google/flatbuffers v2.0.8+incompatible go: downloading github.com/goccy/go-json v0.10.2 go: downloading github.com/zeebo/xxh3 v1.0.2 go: downloading github.com/andybalholm/brotli v1.0.5 go: downloading github.com/klauspost/cpuid/v2 v2.2.4 found first match line: 457152668008448017 : [2025/04/06 08:23:53.526 +08:00] [INFO] [ddl_manager.go:298] ["handle a ddl job"] [namespace=default] [changefeed=changefeed-for-find-finished-ts] [tableID=172] [jobID=173] [query="CREATE TABLE IF NOT EXISTS `finishmark19000` (`foo` BIGINT PRIMARY KEY)"] [finishedTs=457152668008448017] [2025/04/06 08:23:56.038 +08:00] [INFO] [case.go:581] ["ddl synced"] [table=finishmark19000] [2025/04/06 08:23:56.038 +08:00] [INFO] [case.go:587] ["downstream sync success"] [endTs=457152668008448017] [2025/04/06 08:23:56.042 +08:00] [INFO] [case.go:191] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2025/04/06 08:23:56.046 +08:00] [INFO] [case.go:191] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2025/04/06 08:23:56.050 +08:00] [INFO] [case.go:308] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2025/04/06 08:23:56.132 +08:00] [INFO] [case.go:308] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2025/04/06 08:23:56.144 +08:00] [INFO] [case.go:561] ["one upstream workload finished"] [round=20001] [2025/04/06 08:23:56.144 +08:00] [INFO] [case.go:608] ["one downstream exit due to receive done"] [2025/04/06 08:23:56.145 +08:00] [INFO] [case.go:561] ["one upstream workload finished"] [round=20002] [2025/04/06 08:23:56.145 +08:00] [INFO] [case.go:608] ["one downstream exit due to receive done"] [2025/04/06 08:23:56.147 +08:00] [INFO] [case.go:561] ["one upstream workload finished"] [round=20003] [2025/04/06 08:23:56.147 +08:00] [INFO] [case.go:608] ["one downstream exit due to receive done"] [2025/04/06 08:23:56.148 +08:00] [INFO] [case.go:561] ["one upstream workload finished"] [round=20004] [2025/04/06 08:23:56.148 +08:00] [INFO] [case.go:608] ["one downstream exit due to receive done"] table test.table5 not exists for 3-th check, retry later nonempty select id, val from test.availability1 where id=1 and val=1 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later [2025/04/06 08:23:56.726 +08:00] [INFO] [case.go:544] ["upstream create table executed"] [tblName=finishmark20000] [2025/04/06 08:23:56.726 +08:00] [INFO] [case.go:561] ["one upstream workload finished"] [round=20004] [2025/04/06 08:23:56.726 +08:00] [INFO] [case.go:576] ["downstream start wait for table"] [tblName=finishmark20000] go: downloading github.com/pierrec/lz4 v2.6.1+incompatible table processor_delay.t46 exists table processor_delay.t47 not exists for 1-th check, retry later check diff successfully check_ts_not_forward ddl-only-block-related-table check diff failed 3-th time, retry later table test.finish_mark not exists for 28-th check, retry later table batch_add_table.finish_mark not exists for 5-th check, retry later nonempty select id, val from test.availability1 where id=1 and val=1 table test.table5 exists table test.table6 not exists for 1-th check, retry later TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 2-th time, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/sql_mode/run.sh using Sink-Type: mysql... <<================= +++ dirname /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/sql_mode/run.sh ++ cd /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/sql_mode ++ pwd + CUR=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/sql_mode + source /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/sql_mode/../_utils/test_prepare ++ UP_TIDB_HOST=127.0.0.1 ++ UP_TIDB_PORT=4000 ++ UP_TIDB_OTHER_PORT=4001 ++ UP_TIDB_STATUS=10080 ++ UP_TIDB_OTHER_STATUS=10081 ++ DOWN_TIDB_HOST=127.0.0.1 ++ DOWN_TIDB_PORT=3306 ++ DOWN_TIDB_STATUS=20080 ++ TLS_TIDB_HOST=127.0.0.1 ++ TLS_TIDB_PORT=3307 ++ TLS_TIDB_STATUS=30080 ++ UP_PD_HOST_1=127.0.0.1 ++ UP_PD_PORT_1=2379 ++ UP_PD_PEER_PORT_1=2380 ++ UP_PD_HOST_2=127.0.0.1 ++ UP_PD_PORT_2=2679 ++ UP_PD_PEER_PORT_2=2680 ++ UP_PD_HOST_3=127.0.0.1 ++ UP_PD_PORT_3=2779 ++ UP_PD_PEER_PORT_3=2780 ++ DOWN_PD_HOST=127.0.0.1 ++ DOWN_PD_PORT=2479 ++ DOWN_PD_PEER_PORT=2480 ++ TLS_PD_HOST=127.0.0.1 ++ TLS_PD_PORT=2579 ++ TLS_PD_PEER_PORT=2580 ++ UP_TIKV_HOST_1=127.0.0.1 ++ UP_TIKV_PORT_1=20160 ++ UP_TIKV_STATUS_PORT_1=20181 ++ UP_TIKV_HOST_2=127.0.0.1 ++ UP_TIKV_PORT_2=20161 ++ UP_TIKV_STATUS_PORT_2=20182 ++ UP_TIKV_HOST_3=127.0.0.1 ++ UP_TIKV_PORT_3=20162 ++ UP_TIKV_STATUS_PORT_3=20183 ++ DOWN_TIKV_HOST=127.0.0.1 ++ DOWN_TIKV_PORT=21160 ++ DOWN_TIKV_STATUS_PORT=21180 ++ TLS_TIKV_HOST=127.0.0.1 ++ TLS_TIKV_PORT=22160 ++ TLS_TIKV_STATUS_PORT=22180 +++ cat /tmp/tidb_cdc_test/KAFKA_VERSION +++ echo 2.4.1 ++ KAFKA_VERSION=2.4.1 + WORK_DIR=/tmp/tidb_cdc_test/sql_mode + CDC_BINARY=cdc.test + SINK_TYPE=mysql + CDC_COUNT=3 + DB_COUNT=4 + rm -rf /tmp/tidb_cdc_test/sql_mode + mkdir -p /tmp/tidb_cdc_test/sql_mode + start_tidb_cluster --workdir /tmp/tidb_cdc_test/sql_mode The 1 times to try to start tidb cluster... [2025/04/06 08:23:58.727 +08:00] [INFO] [case.go:687] ["try get end ts"] [query="SELECT JOB_ID FROM information_schema.ddl_jobs WHERE table_name = ?"] [tableName=finishmark20000] [2025/04/06 08:23:58.734 +08:00] [INFO] [case.go:697] ["try parse finishedTs from ticdc log"] [tableName=finishmark20000] /tmp/tidb_cdc_test/bank/cdcdown.log [2025/04/06 08:23:58.735 +08:00] [INFO] [case.go:717] ["total files"] [file="[/tmp/tidb_cdc_test/bank/cdcdown.log]"] found first match line: 457152669314973710 : [2025/04/06 08:23:58.464 +08:00] [INFO] [ddl_manager.go:298] ["handle a ddl job"] [namespace=default] [changefeed=changefeed-for-find-finished-ts] [tableID=174] [jobID=175] [query="CREATE TABLE IF NOT EXISTS `finishmark20000` (`foo` BIGINT PRIMARY KEY)"] [finishedTs=457152669314973710] [2025/04/06 08:23:58.967 +08:00] [INFO] [case.go:581] ["ddl synced"] [table=finishmark20000] [2025/04/06 08:23:58.967 +08:00] [INFO] [case.go:587] ["downstream sync success"] [endTs=457152669314973710] [2025/04/06 08:23:58.970 +08:00] [INFO] [case.go:191] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2025/04/06 08:23:58.974 +08:00] [INFO] [case.go:191] ["sequence verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2025/04/06 08:23:58.979 +08:00] [INFO] [case.go:308] ["bank verify pass"] [tag=root@tcp(127.0.0.1:4000)/bank] [2025/04/06 08:23:58.984 +08:00] [INFO] [case.go:308] ["bank verify pass"] [tag=root@tcp(127.0.0.1:3306)/bank] [2025/04/06 08:23:58.984 +08:00] [INFO] [case.go:608] ["one downstream exit due to receive done"] [2025/04/06 08:23:58.984 +08:00] [INFO] [case.go:620] ["bank test finished"] [valid=20] [tries=20] [ratio=1] + cleanup_process cdc.test wait process cdc.test exit for 1-th time... table processor_delay.t47 exists table processor_delay.t48 not exists for 1-th check, retry later check diff failed 4-th time, retry later table test.finish_mark not exists for 29-th check, retry later table batch_add_table.finish_mark not exists for 6-th check, retry later go: downloading go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 go: downloading github.com/googleapis/enterprise-certificate-proxy v0.3.2 go: downloading github.com/google/s2a-go v0.1.7 go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 go: downloading github.com/felixge/httpsnoop v1.0.4 go: downloading github.com/jmespath/go-jmespath v0.4.0 wait process cdc.test exit for 2-th time... table test.table6 exists check diff successfully Resume changefeed with checkpointTs1 457152665901334530 wait process cdc.test exit for 3-th time... table test.table3 not exists for 1-th check, retry later cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit ++ date + echo '[Sun Apr 6 08:24:01 CST 2025] <<<<<< run test case bank success! >>>>>>' [Sun Apr 6 08:24:01 CST 2025] <<<<<< run test case bank success! >>>>>> + stop_tidb_cluster table processor_delay.t48 exists table processor_delay.t49 not exists for 1-th check, retry later table test.finish_mark not exists for 30-th check, retry later check diff failed 5-th time, retry later table batch_add_table.finish_mark not exists for 7-th check, retry later nonempty select id, val from test.availability1 where id=1 and val=1 run task successfully nonempty select id, val from test.availability1 where id=1 and val=22 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later start tidb cluster in /tmp/tidb_cdc_test/sql_mode Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... table test.table3 not exists for 2-th check, retry later table processor_delay.t49 exists table processor_delay.t50 not exists for 1-th check, retry later table test.finish_mark not exists for 31-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/consistent_replicate_storage_file/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/capture_session_done_during_task/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... check diff successfully table batch_add_table.finish_mark not exists for 8-th check, retry later nonempty select id, val from test.availability1 where id=1 and val=22 run task successfully empty select id, val from test.availability1 where id=1 TEST FAILED: OUTPUT CONTAINS 'id:' ____________________________________ *************************** 1. row *************************** id: 1 val: 22 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later Verifying downstream PD is started... ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 221 100 221 0 0 7148 0 --:--:-- --:--:-- --:--:-- 7366 + synced_status='{"synced":true,"sink_checkpoint_ts":"2025-04-06 08:23:51.309","puller_resolved_ts":"2025-04-06 08:23:44.309","last_synced_ts":"2025-04-06 08:21:35.959","now_ts":"2025-04-06 08:23:53.000","info":"Data syncing is finished"}' ++ echo '{"synced":true,"sink_checkpoint_ts":"2025-04-06' '08:23:51.309","puller_resolved_ts":"2025-04-06' '08:23:44.309","last_synced_ts":"2025-04-06' '08:21:35.959","now_ts":"2025-04-06' '08:23:53.000","info":"Data' syncing is 'finished"}' ++ jq .synced + status=true + '[' true '!=' true ']' + kill_pd ++ ps aux ++ grep pd-server ++ grep /tmp/tidb_cdc_test/synced_status_with_redo + info='jenkins 9249 9.8 0.0 14485656 154160 ? Sl 08:21 0:15 pd-server --advertise-client-urls http://127.0.0.1:2379 --client-urls http://0.0.0.0:2379 --advertise-peer-urls http://127.0.0.1:2380 --peer-urls http://0.0.0.0:2380 --config /tmp/tidb_cdc_test/synced_status_with_redo/pd-config.toml --log-file /tmp/tidb_cdc_test/synced_status_with_redo/pd1.log --data-dir /tmp/tidb_cdc_test/synced_status_with_redo/pd1 --name=pd1 --initial-cluster=pd1=http://127.0.0.1:2380 jenkins 9299 7.3 0.0 14484376 143992 ? Sl 08:21 0:11 pd-server --advertise-client-urls http://127.0.0.1:2479 --client-urls http://0.0.0.0:2479 --advertise-peer-urls http://127.0.0.1:2480 --peer-urls http://0.0.0.0:2480 --config /tmp/tidb_cdc_test/synced_status_with_redo/pd-config.toml --log-file /tmp/tidb_cdc_test/synced_status_with_redo/down_pd.log --data-dir /tmp/tidb_cdc_test/synced_status_with_redo/down_pd' ++ ps aux ++ grep pd-server ++ grep /tmp/tidb_cdc_test/synced_status_with_redo ++ awk '{print $2}' ++ xargs kill -9 + sleep 20 {"level":"warn","ts":1743899039.7841034,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc004c516c0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":1743899039.7841809,"caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":1743899039.8108454,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc002eb6380/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":1743899039.8109126,"caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":1743899039.8214462,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0053fd880/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} {"level":"info","ts":1743899039.821494,"caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table test.table3 not exists for 3-th check, retry later check diff failed 1-th time, retry later table processor_delay.t50 exists check diff failed 1-th time, retry later table test.finish_mark not exists for 32-th check, retry later table batch_add_table.finish_mark not exists for 9-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/consistent_replicate_storage_file Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) empty select id, val from test.availability1 where id=1 run task successfully nonempty select id, val from test.availability2 where id=1 and val=1 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later table test.table3 exists table test.table4 not exists for 1-th check, retry later check diff failed 2-th time, retry later \033[0;36m<<< Run all test success >>>\033[0m {"level":"warn","ts":"2025-04-06T08:24:07.119976+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001cdd340/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2025-04-06T08:24:07.120861+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000985a40/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} [Pipeline] } [Pipeline] // timeout [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-periodics_cdc_integration_mysql_test-234/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } {"level":"warn","ts":"2025-04-06T08:24:07.213714+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc002085500/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} [Pipeline] // dir [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv table test.finish_mark exists [Pipeline] } table batch_add_table.finish_mark not exists for 10-th check, retry later [Pipeline] // stage [Pipeline] } check diff successfully wait process cdc.test exit for 1-th time... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) nonempty select id, val from test.availability2 where id=1 and val=1 run task successfully nonempty select id, val from test.availability2 where id=1 and val=22 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later table test.table4 exists start tidb cluster in /tmp/tidb_cdc_test/capture_session_done_during_task Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... check diff failed 1-th time, retry later check diff successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... check diff failed 3-th time, retry later TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... table batch_add_table.finish_mark not exists for 11-th check, retry later cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Sun Apr 6 08:24:10 CST 2025] <<<<<< run test case processor_etcd_worker_delay success! >>>>>> cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Sun Apr 6 08:24:10 CST 2025] <<<<<< run test case tidb_mysql_test success! >>>>>> nonempty select id, val from test.availability2 where id=1 and val=22 run task successfully empty select id, val from test.availability2 where id=1 TEST FAILED: OUTPUT CONTAINS 'id:' ____________________________________ *************************** 1. row *************************** id: 1 val: 22 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489989357300557951 TiDB Cluster ID. tikv_gc_leader_uuid 65821ef08600007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-j-shst4, pid:9157, start at 2025-04-06 08:24:10.546273365 +0800 CST m=+3.817043453 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:26:10.556 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:24:10.520 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:14:10.520 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489989357300557951 TiDB Cluster ID. tikv_gc_leader_uuid 65821ef08600007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-j-shst4, pid:9157, start at 2025-04-06 08:24:10.546273365 +0800 CST m=+3.817043453 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:26:10.556 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:24:10.520 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:14:10.520 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489989362460687017 TiDB Cluster ID. tikv_gc_leader_uuid 65821ef09fc0002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-j-shst4, pid:9225, start at 2025-04-06 08:24:10.627926606 +0800 CST m=+3.832542938 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:26:10.644 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:24:10.623 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:14:10.623 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/sql_mode/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/sql_mode/tiflash/log/error.log arg matches is ArgMatches { args: {"memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/sql_mode/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/sql_mode/tiflash-proxy.toml"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/sql_mode/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } check diff failed 4-th time, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore check diff successfully wait process cdc.test exit for 1-th time... Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 2-th time... table batch_add_table.finish_mark not exists for 12-th check, retry later cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Sun Apr 6 08:24:12 CST 2025] <<<<<< START cdc server in changefeed_resume_with_checkpoint_ts case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/InjectChangefeedFastFailError=1*return(true)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_resume_with_checkpoint_ts.87328734.out server --log-file /tmp/tidb_cdc_test/changefeed_resume_with_checkpoint_ts/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_resume_with_checkpoint_ts/cdc_data --cluster-id default --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 empty select id, val from test.availability2 where id=1 run task successfully nonempty select id, val from test.availability3 where id=1 and val=1 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later + trap stop_tidb_cluster EXIT + run_sql 'set global sql_mode='\''NO_BACKSLASH_ESCAPES'\'';' 127.0.0.1 4000 + run_sql 'set global sql_mode='\''NO_BACKSLASH_ESCAPES'\'';' 127.0.0.1 3306 check diff failed 5-th time, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/kill_owner_with_ddl/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + cd /tmp/tidb_cdc_test/sql_mode ++ run_cdc_cli_tso_query 127.0.0.1 2379 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sql_mode.cli.10541.out cli tso query --pd=http://127.0.0.1:2379 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) {"level":"warn","ts":1743899052.8716183,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc004c516c0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":1743899052.9059079,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0053fd880/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} {"level":"warn","ts":"2025-04-06T08:24:13.121275+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001cdd340/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2025-04-06T08:24:13.122642+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000985a40/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0{"level":"warn","ts":"2025-04-06T08:24:13.215178+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc002085500/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} table batch_add_table.finish_mark not exists for 13-th check, retry later nonempty select id, val from test.availability3 where id=1 and val=1 run task successfully nonempty select id, val from test.availability3 where id=1 and val=22 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later 0 0 0 0 0 0 0 0 --:--:-- 0:00:01 --:--:-- 0 check diff successfully ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x + tso='457152673551482881 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 457152673551482881 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + start_ts=457152673551482881 + run_cdc_server --workdir /tmp/tidb_cdc_test/sql_mode --binary cdc.test [Sun Apr 6 08:24:15 CST 2025] <<<<<< START cdc server in sql_mode case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sql_mode.1057710579.out server --log-file /tmp/tidb_cdc_test/sql_mode/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/sql_mode/cdc_data --cluster-id default + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2025/04/06 08:24:10.744 +08:00] [INFO] [case.go:115] ["sync updatePKUK take: 12.622825268s"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 06 Apr 2025 00:24:15 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ab5739de-fbf5-490c-8963-0dfadff1b907 {"id":"ab5739de-fbf5-490c-8963-0dfadff1b907","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899053} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 3 /tidb/cdc/default/__cdc_meta__/owner/223196087b209ea2 ab5739de-fbf5-490c-8963-0dfadff1b907 /tidb/cdc/default/default/changefeed/info/2ca3e314-fdba-4651-a492-189ce147c35a {"upstream-id":7489989219320878993,"namespace":"default","changefeed-id":"2ca3e314-fdba-4651-a492-189ce147c35a","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2025-04-06T08:23:44.475454946+08:00","start-ts":457152665901334530,"target-ts":0,"admin-job-type":1,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"failed","error":{"time":"2025-04-06T08:24:13.213416941+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrStartTsBeforeGC","message":"[CDC:ErrStartTsBeforeGC]InjectChangefeedFastFailError"},"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457152673398915079} /tidb/cdc/default/default/changefeed/status/2ca3e314-fdba-4651-a492-189ce147c35a {"checkpoint-ts":457152667762556942,"min-table-barrier-ts":457152667762556942,"admin-job-type":1} /tidb/cdc/default/default/upstream/7489989219320878993 {"id":7489989219320878993,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ab5739de-fbf5-490c-8963-0dfadff1b907 {"id":"ab5739de-fbf5-490c-8963-0dfadff1b907","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899053} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 3 /tidb/cdc/default/__cdc_meta__/owner/223196087b209ea2 ab5739de-fbf5-490c-8963-0dfadff1b907 /tidb/cdc/default/default/changefeed/info/2ca3e314-fdba-4651-a492-189ce147c35a {"upstream-id":7489989219320878993,"namespace":"default","changefeed-id":"2ca3e314-fdba-4651-a492-189ce147c35a","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2025-04-06T08:23:44.475454946+08:00","start-ts":457152665901334530,"target-ts":0,"admin-job-type":1,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"failed","error":{"time":"2025-04-06T08:24:13.213416941+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrStartTsBeforeGC","message":"[CDC:ErrStartTsBeforeGC]InjectChangefeedFastFailError"},"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457152673398915079} /tidb/cdc/default/default/changefeed/status/2ca3e314-fdba-4651-a492-189ce147c35a {"checkpoint-ts":457152667762556942,"min-table-barrier-ts":457152667762556942,"admin-job-type":1} /tidb/cdc/default/default/upstream/7489989219320878993 {"id":7489989219320878993,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ab5739de-fbf5-490c-8963-0dfadff1b907 {"id":"ab5739de-fbf5-490c-8963-0dfadff1b907","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899053} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 3 /tidb/cdc/default/__cdc_meta__/owner/223196087b209ea2 ab5739de-fbf5-490c-8963-0dfadff1b907 /tidb/cdc/default/default/changefeed/info/2ca3e314-fdba-4651-a492-189ce147c35a {"upstream-id":7489989219320878993,"namespace":"default","changefeed-id":"2ca3e314-fdba-4651-a492-189ce147c35a","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2025-04-06T08:23:44.475454946+08:00","start-ts":457152665901334530,"target-ts":0,"admin-job-type":1,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"failed","error":{"time":"2025-04-06T08:24:13.213416941+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrStartTsBeforeGC","message":"[CDC:ErrStartTsBeforeGC]InjectChangefeedFastFailError"},"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457152673398915079} /tidb/cdc/default/default/changefeed/status/2ca3e314-fdba-4651-a492-189ce147c35a {"checkpoint-ts":457152667762556942,"min-table-barrier-ts":457152667762556942,"admin-job-type":1} /tidb/cdc/default/default/upstream/7489989219320878993 {"id":7489989219320878993,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x check_changefeed_state http://127.0.0.1:2379 2ca3e314-fdba-4651-a492-189ce147c35a failed ErrStartTsBeforeGC + endpoints=http://127.0.0.1:2379 + changefeed_id=2ca3e314-fdba-4651-a492-189ce147c35a + expected_state=failed + error_msg=ErrStartTsBeforeGC + tls_dir=ErrStartTsBeforeGC + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c 2ca3e314-fdba-4651-a492-189ce147c35a -s + info='{ "upstream_id": 7489989219320878993, "namespace": "default", "id": "2ca3e314-fdba-4651-a492-189ce147c35a", "state": "failed", "checkpoint_tso": 457152667762556942, "checkpoint_time": "2025-04-06 08:23:51.687", "error": { "time": "2025-04-06T08:24:13.213416941+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrStartTsBeforeGC", "message": "[CDC:ErrStartTsBeforeGC]InjectChangefeedFastFailError" } }' + echo '{ "upstream_id": 7489989219320878993, "namespace": "default", "id": "2ca3e314-fdba-4651-a492-189ce147c35a", "state": "failed", "checkpoint_tso": 457152667762556942, "checkpoint_time": "2025-04-06 08:23:51.687", "error": { "time": "2025-04-06T08:24:13.213416941+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrStartTsBeforeGC", "message": "[CDC:ErrStartTsBeforeGC]InjectChangefeedFastFailError" } }' { "upstream_id": 7489989219320878993, "namespace": "default", "id": "2ca3e314-fdba-4651-a492-189ce147c35a", "state": "failed", "checkpoint_tso": 457152667762556942, "checkpoint_time": "2025-04-06 08:23:51.687", "error": { "time": "2025-04-06T08:24:13.213416941+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrStartTsBeforeGC", "message": "[CDC:ErrStartTsBeforeGC]InjectChangefeedFastFailError" } } ++ echo '{' '"upstream_id":' 7489989219320878993, '"namespace":' '"default",' '"id":' '"2ca3e314-fdba-4651-a492-189ce147c35a",' '"state":' '"failed",' '"checkpoint_tso":' 457152667762556942, '"checkpoint_time":' '"2025-04-06' '08:23:51.687",' '"error":' '{' '"time":' '"2025-04-06T08:24:13.213416941+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrStartTsBeforeGC",' '"message":' '"[CDC:ErrStartTsBeforeGC]InjectChangefeedFastFailError"' '}' '}' ++ jq -r .state + state=failed + [[ ! failed == \f\a\i\l\e\d ]] ++ jq -r .error.message ++ echo '{' '"upstream_id":' 7489989219320878993, '"namespace":' '"default",' '"id":' '"2ca3e314-fdba-4651-a492-189ce147c35a",' '"state":' '"failed",' '"checkpoint_tso":' 457152667762556942, '"checkpoint_time":' '"2025-04-06' '08:23:51.687",' '"error":' '{' '"time":' '"2025-04-06T08:24:13.213416941+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrStartTsBeforeGC",' '"message":' '"[CDC:ErrStartTsBeforeGC]InjectChangefeedFastFailError"' '}' '}' wait process cdc.test exit for 1-th time... table batch_add_table.finish_mark not exists for 14-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489989380902714499 TiDB Cluster ID. tikv_gc_leader_uuid 65821ef3da40002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-6-zhp59, pid:9846, start at 2025-04-06 08:24:13.933462862 +0800 CST m=+2.274377391 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:26:13.944 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:24:13.949 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:14:13.949 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489989380902714499 TiDB Cluster ID. tikv_gc_leader_uuid 65821ef3da40002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-6-zhp59, pid:9846, start at 2025-04-06 08:24:13.933462862 +0800 CST m=+2.274377391 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:26:13.944 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:24:13.949 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:14:13.949 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489989371177427213 TiDB Cluster ID. tikv_gc_leader_uuid 65821ef3f700003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-6-zhp59, pid:9911, start at 2025-04-06 08:24:14.047535242 +0800 CST m=+2.309083960 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:26:14.055 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:24:14.044 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:14:14.044 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/consistent_replicate_storage_file/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/consistent_replicate_storage_file/tiflash/log/error.log arg matches is ArgMatches { args: {"memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/consistent_replicate_storage_file/tiflash-proxy.toml"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/consistent_replicate_storage_file/tiflash/log/proxy.log"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/consistent_replicate_storage_file/tiflash/db/proxy"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + message='[CDC:ErrStartTsBeforeGC]InjectChangefeedFastFailError' + [[ ! [CDC:ErrStartTsBeforeGC]InjectChangefeedFastFailError =~ ErrStartTsBeforeGC ]] run task successfully 0 0 0 0 0 0 0 0 --:--:-- 0:00:02 --:--:-- 0 check_changefeed_state http://127.0.0.1:2379 2ca3e314-fdba-4651-a492-189ce147c35a normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=2ca3e314-fdba-4651-a492-189ce147c35a + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c 2ca3e314-fdba-4651-a492-189ce147c35a -s wait process cdc.test exit for 2-th time... + info='{ "upstream_id": 7489989219320878993, "namespace": "default", "id": "2ca3e314-fdba-4651-a492-189ce147c35a", "state": "normal", "checkpoint_tso": 457152674211299330, "checkpoint_time": "2025-04-06 08:24:16.287", "error": null }' + echo '{ "upstream_id": 7489989219320878993, "namespace": "default", "id": "2ca3e314-fdba-4651-a492-189ce147c35a", "state": "normal", "checkpoint_tso": 457152674211299330, "checkpoint_time": "2025-04-06 08:24:16.287", "error": null }' { "upstream_id": 7489989219320878993, "namespace": "default", "id": "2ca3e314-fdba-4651-a492-189ce147c35a", "state": "normal", "checkpoint_tso": 457152674211299330, "checkpoint_time": "2025-04-06 08:24:16.287", "error": null } ++ echo '{' '"upstream_id":' 7489989219320878993, '"namespace":' '"default",' '"id":' '"2ca3e314-fdba-4651-a492-189ce147c35a",' '"state":' '"normal",' '"checkpoint_tso":' 457152674211299330, '"checkpoint_time":' '"2025-04-06' '08:24:16.287",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7489989219320878993, '"namespace":' '"default",' '"id":' '"2ca3e314-fdba-4651-a492-189ce147c35a",' '"state":' '"normal",' '"checkpoint_tso":' 457152674211299330, '"checkpoint_time":' '"2025-04-06' '08:24:16.287",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully nonempty select id, val from test.availability3 where id=1 and val=22 run task successfully empty select id, val from test.availability3 where id=1 TEST FAILED: OUTPUT CONTAINS 'id:' ____________________________________ *************************** 1. row *************************** id: 1 val: 22 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later table test1.table1 not exists for 1-th check, retry later cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Sun Apr 6 08:24:17 CST 2025] <<<<<< run test case changefeed_pause_resume success! >>>>>> 0 0 0 0 0 0 0 0 --:--:-- 0:00:03 --:--:-- 0 start tidb cluster in /tmp/tidb_cdc_test/kill_owner_with_ddl Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489989388887227743 TiDB Cluster ID. tikv_gc_leader_uuid 65821ef59c80009 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-4-tjmd8, pid:22109, start at 2025-04-06 08:24:15.745024661 +0800 CST m=+2.366937075 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:26:15.753 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:24:15.730 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:14:15.730 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489989388887227743 TiDB Cluster ID. tikv_gc_leader_uuid 65821ef59c80009 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-4-tjmd8, pid:22109, start at 2025-04-06 08:24:15.745024661 +0800 CST m=+2.366937075 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:26:15.753 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:24:15.730 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:14:15.730 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489989389411717317 TiDB Cluster ID. tikv_gc_leader_uuid 65821ef5838000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-4-tjmd8, pid:22181, start at 2025-04-06 08:24:15.661300872 +0800 CST m=+2.221611321 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:26:15.668 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:24:15.666 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:14:15.666 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/capture_session_done_during_task/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/capture_session_done_during_task/tiflash/log/error.log arg matches is ArgMatches { args: {"addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/capture_session_done_during_task/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/capture_session_done_during_task/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/capture_session_done_during_task/tiflash/log/proxy.log"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table batch_add_table.finish_mark not exists for 15-th check, retry later [Sun Apr 6 08:24:18 CST 2025] <<<<<< START cdc server in consistent_replicate_storage_file case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.consistent_replicate_storage_file.1116811170.out server --log-file /tmp/tidb_cdc_test/consistent_replicate_storage_file/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/consistent_replicate_storage_file/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 0 0 0 0 0 0 0 0 --:--:-- 0:00:04 --:--:-- 0 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 06 Apr 2025 00:24:18 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/623ac603-413a-4a2c-99ce-f293ec175a86 {"id":"623ac603-413a-4a2c-99ce-f293ec175a86","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899055} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087ba13feb 623ac603-413a-4a2c-99ce-f293ec175a86 /tidb/cdc/default/default/upstream/7489989357300557951 {"id":7489989357300557951,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/623ac603-413a-4a2c-99ce-f293ec175a86 {"id":"623ac603-413a-4a2c-99ce-f293ec175a86","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899055} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087ba13feb 623ac603-413a-4a2c-99ce-f293ec175a86 /tidb/cdc/default/default/upstream/7489989357300557951 {"id":7489989357300557951,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/623ac603-413a-4a2c-99ce-f293ec175a86 {"id":"623ac603-413a-4a2c-99ce-f293ec175a86","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899055} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087ba13feb 623ac603-413a-4a2c-99ce-f293ec175a86 /tidb/cdc/default/default/upstream/7489989357300557951 {"id":7489989357300557951,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + SINK_URI='mysql://root@127.0.0.1:3306/?max-txn-row=1' + run_cdc_cli changefeed create --start-ts=457152673551482881 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sql_mode.cli.10634.out cli changefeed create --start-ts=457152673551482881 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 empty select id, val from test.availability3 where id=1 table test1.table1 not exists for 2-th check, retry later Create changefeed successfully! ID: test-1 Info: {"upstream_id":7489989357300557951,"namespace":"default","id":"test-1","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2025-04-06T08:24:18.991219273+08:00","start_ts":457152673551482881,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457152673551482881,"checkpoint_ts":457152673551482881,"checkpoint_time":"2025-04-06 08:24:13.770"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... run task successfully delete_owner_key pass 0 0 0 0 0 0 0 0 --:--:-- 0:00:05 --:--:-- 0{"level":"warn","ts":"2025-04-06T08:24:19.12233+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001cdd340/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2025-04-06T08:24:19.124097+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000985a40/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} 0 0 0 0 0 0 0 0 --:--:-- 0:00:06 --:--:-- 0{"level":"warn","ts":"2025-04-06T08:24:19.216445+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc002085500/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... table batch_add_table.finish_mark not exists for 16-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/sink_hang/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/ddl_reentrant/run.sh using Sink-Type: mysql... <<================= ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) The 1 times to try to start tidb cluster... + set +x + run_sql 'use test; create table t1(id bigint primary key, a text, b text as ((regexp_replace(a, '\''^[1-9]\d{9,29}$'\'', '\''aaaaa'\''))), c text); insert into t1 (id, a, c) values(1,123456, '\''ab\\\\c'\''); insert into t1 (id, a, c) values(2,1234567890123, '\''ab\\c'\'');' 127.0.0.1 4000 wait process cdc.test exit for 3-th time... process cdc.test already exit run test case test_resign_owner [Sun Apr 6 08:24:20 CST 2025] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.55075509.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_resign_owner.server1.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_resign_owner.server1 --cluster-id default --addr 127.0.0.1:8300 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + '[' mysql == mysql ']' + check_table_exists test.t1 127.0.0.1 3306 table test.t1 not exists for 1-th check, retry later + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.capture_session_done_during_task.cli.23588.out cli tso query --pd=http://127.0.0.1:2379 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 06 Apr 2025 00:24:21 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7043922c-3456-4649-a611-ed72217d37bc {"id":"7043922c-3456-4649-a611-ed72217d37bc","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899058} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087bb09bd6 7043922c-3456-4649-a611-ed72217d37bc /tidb/cdc/default/default/upstream/7489989380902714499 {"id":7489989380902714499,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7043922c-3456-4649-a611-ed72217d37bc {"id":"7043922c-3456-4649-a611-ed72217d37bc","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899058} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087bb09bd6 7043922c-3456-4649-a611-ed72217d37bc /tidb/cdc/default/default/upstream/7489989380902714499 {"id":7489989380902714499,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7043922c-3456-4649-a611-ed72217d37bc {"id":"7043922c-3456-4649-a611-ed72217d37bc","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899058} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087bb09bd6 7043922c-3456-4649-a611-ed72217d37bc /tidb/cdc/default/default/upstream/7489989380902714499 {"id":7489989380902714499,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x table test1.table1 exists table test1.table2 not exists for 1-th check, retry later 0 0 0 0 0 0 0 0 --:--:-- 0:00:07 --:--:-- 0 Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ***************** properties ***************** "operationcount"="0" "mysql.host"="127.0.0.1" "mysql.port"="4000" "updateproportion"="0" "recordcount"="5000" "insertproportion"="0" "workload"="core" "mysql.user"="root" "scanproportion"="0" "requestdistribution"="uniform" "readproportion"="0" "dotransactions"="false" "mysql.db"="consistent_replicate_storage_file" "threadcount"="10" "readallfields"="true" "fieldcount"="100" ********************************************** table batch_add_table.finish_mark not exists for 17-th check, retry later 0 0 0 0 0 0 0 0 --:--:-- 0:00:08 --:--:-- 0 + set +x + tso='457152675517038593 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 457152675517038593 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Sun Apr 6 08:24:22 CST 2025] <<<<<< START cdc server in capture_session_done_during_task case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/processor/processorManagerHandleNewChangefeedDelay=sleep(2000)' + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.capture_session_done_during_task.2363923641.out server --log-file /tmp/tidb_cdc_test/capture_session_done_during_task/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/capture_session_done_during_task/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 start tidb cluster in /tmp/tidb_cdc_test/sink_hang Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... table test1.table2 exists check diff successfully 0 0 0 0 0 0 0 0 --:--:-- 0:00:09 --:--:-- 0 table test.t1 exists + sleep 10 [2025/04/06 08:24:18.476 +08:00] [INFO] [pd_service_discovery.go:1025] ["[pd] switch leader"] [new-leader=http://127.0.0.1:2379] [old-leader=] [2025/04/06 08:24:18.476 +08:00] [INFO] [pd_service_discovery.go:499] ["[pd] init cluster id"] [cluster-id=7489988645087467134] [2025/04/06 08:24:18.477 +08:00] [INFO] [client.go:532] ["[pd] changing service mode"] [old-mode=UNKNOWN_SVC_MODE] [new-mode=PD_SVC_MODE] [2025/04/06 08:24:18.477 +08:00] [INFO] [tso_client.go:296] ["[tso] switch dc tso global allocator serving url"] [dc-location=global] [new-url=http://127.0.0.1:2379] [2025/04/06 08:24:18.477 +08:00] [INFO] [client.go:538] ["[pd] service mode changed"] [old-mode=UNKNOWN_SVC_MODE] [new-mode=PD_SVC_MODE] [2025/04/06 08:24:18.477 +08:00] [INFO] [tso_dispatcher.go:140] ["[tso] start tso deadline watcher"] [dc-location=global] [2025/04/06 08:24:18.477 +08:00] [INFO] [tso_dispatcher.go:198] ["[tso] tso dispatcher created"] [dc-location=global] [2025/04/06 08:24:18.477 +08:00] [INFO] [tso_client.go:132] ["[tso] start tso dispatcher check loop"] [2025/04/06 08:24:18.477 +08:00] [INFO] [tso_dispatcher.go:723] ["[tso] switching tso rpc concurrency"] [old=0] [new=1] [2025/04/06 08:24:18.477 +08:00] [INFO] [tso_dispatcher.go:476] ["[tso] start tso connection contexts updater"] [dc-location=global] [2025/04/06 08:24:18.478 +08:00] [INFO] [pd_service_discovery.go:1025] ["[pd] switch leader"] [new-leader=http://127.0.0.1:2379] [old-leader=] [2025/04/06 08:24:18.478 +08:00] [INFO] [pd_service_discovery.go:499] ["[pd] init cluster id"] [cluster-id=7489988645087467134] [2025/04/06 08:24:18.478 +08:00] [INFO] [client.go:532] ["[pd] changing service mode"] [old-mode=UNKNOWN_SVC_MODE] [new-mode=PD_SVC_MODE] [2025/04/06 08:24:18.478 +08:00] [INFO] [tso_client.go:296] ["[tso] switch dc tso global allocator serving url"] [dc-location=global] [new-url=http://127.0.0.1:2379] [2025/04/06 08:24:18.479 +08:00] [INFO] [client.go:538] ["[pd] service mode changed"] [old-mode=UNKNOWN_SVC_MODE] [new-mode=PD_SVC_MODE] [2025/04/06 08:24:18.479 +08:00] [INFO] [tso_dispatcher.go:140] ["[tso] start tso deadline watcher"] [dc-location=global] [2025/04/06 08:24:18.479 +08:00] [INFO] [tso_client.go:132] ["[tso] start tso dispatcher check loop"] [2025/04/06 08:24:18.479 +08:00] [INFO] [tso_dispatcher.go:198] ["[tso] tso dispatcher created"] [dc-location=global] [2025/04/06 08:24:18.479 +08:00] [INFO] [tso_dispatcher.go:723] ["[tso] switching tso rpc concurrency"] [old=0] [new=1] [2025/04/06 08:24:18.479 +08:00] [INFO] [tso_dispatcher.go:476] ["[tso] start tso connection contexts updater"] [dc-location=global] [2025/04/06 08:24:18.479 +08:00] [INFO] [tikv_driver.go:201] ["using API V1."] [2025/04/06 08:24:18.480 +08:00] [INFO] [main.go:180] ["genLock started"] [2025/04/06 08:24:18.481 +08:00] [INFO] [store_cache.go:532] ["change store resolve state"] [store=1] [addr=127.0.0.1:20161] [from=unresolved] [to=resolved] [liveness-state=reachable] [2025/04/06 08:24:20.480 +08:00] [INFO] [pd.go:441] ["adaptive update ts interval state transition"] [configuredInterval=2s] [prevAdaptiveUpdateInterval=2s] [newAdaptiveUpdateInterval=2s] [requiredStaleness=0s] [prevState=unknown(0)] [newState=normal] start tidb cluster in /tmp/tidb_cdc_test/ddl_reentrant Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 06 Apr 2025 00:24:23 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/e684662d-022f-4a79-9ef3-0a59dba96193 {UpstreamID:7489988628380945915 Namespace:default ID:e684662d-022f-4a79-9ef3-0a59dba96193 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-06 08:21:28.032207553 +0800 CST StartTs:457152628763131907 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003b12d80 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457152630100066306} {CheckpointTs:457152674782511112 MinTableBarrierTs:457152675831087109 AdminJobType:noop} span: {tableID:112, startKey:7480000000000000ff705f720000000000fa, endKey:7480000000000000ff705f730000000000fa}, resolvedTs: 457152674782511112, checkpointTs: 457152674782511112, state: Preparing span: {tableID:116, startKey:7480000000000000ff745f720000000000fa, endKey:7480000000000000ff745f730000000000fa}, resolvedTs: 457152674782511112, checkpointTs: 457152674782511112, state: Preparing span: {tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}, resolvedTs: 457152674782511112, checkpointTs: 457152674782511112, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/97d46017-30ff-4beb-994a-38d0f5e943e4 {"id":"97d46017-30ff-4beb-994a-38d0f5e943e4","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899060} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 2 /tidb/cdc/default/__cdc_meta__/owner/2231960879054d4e 97d46017-30ff-4beb-994a-38d0f5e943e4 /tidb/cdc/default/default/changefeed/info/e684662d-022f-4a79-9ef3-0a59dba96193 {"upstream-id":7489988628380945915,"namespace":"default","changefeed-id":"e684662d-022f-4a79-9ef3-0a59dba96193","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-06T08:21:28.032207553+08:00","start-ts":457152628763131907,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457152630100066306} /tidb/cdc/default/default/changefeed/status/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":457152674782511112,"min-table-barrier-ts":457152676092968965,"admin-job-type":0} /tidb/cdc/default/default/task/position/97d46017-30ff-4beb-994a-38d0f5e943e4/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489988628380945915 {"id":7489988628380945915,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/e684662d-022f-4a79-9ef3-0a59dba96193 {UpstreamID:7489988628380945915 Namespace:default ID:e684662d-022f-4a79-9ef3-0a59dba96193 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-06 08:21:28.032207553 +0800 CST StartTs:457152628763131907 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003b12d80 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457152630100066306} {CheckpointTs:457152674782511112 MinTableBarrierTs:457152675831087109 AdminJobType:noop} span: {tableID:112, startKey:7480000000000000ff705f720000000000fa, endKey:7480000000000000ff705f730000000000fa}, resolvedTs: 457152674782511112, checkpointTs: 457152674782511112, state: Preparing span: {tableID:116, startKey:7480000000000000ff745f720000000000fa, endKey:7480000000000000ff745f730000000000fa}, resolvedTs: 457152674782511112, checkpointTs: 457152674782511112, state: Preparing span: {tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}, resolvedTs: 457152674782511112, checkpointTs: 457152674782511112, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/97d46017-30ff-4beb-994a-38d0f5e943e4 {"id":"97d46017-30ff-4beb-994a-38d0f5e943e4","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899060} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 2 /tidb/cdc/default/__cdc_meta__/owner/2231960879054d4e 97d46017-30ff-4beb-994a-38d0f5e943e4 /tidb/cdc/default/default/changefeed/info/e684662d-022f-4a79-9ef3-0a59dba96193 {"upstream-id":7489988628380945915,"namespace":"default","changefeed-id":"e684662d-022f-4a79-9ef3-0a59dba96193","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-06T08:21:28.032207553+08:00","start-ts":457152628763131907,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457152630100066306} /tidb/cdc/default/default/changefeed/status/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":457152674782511112,"min-table-barrier-ts":457152676092968965,"admin-job-type":0} + grep -q 'failed to get info:' /tidb/cdc/default/default/task/position/97d46017-30ff-4beb-994a-38d0f5e943e4/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489988628380945915 {"id":7489988628380945915,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/e684662d-022f-4a79-9ef3-0a59dba96193 {UpstreamID:7489988628380945915 Namespace:default ID:e684662d-022f-4a79-9ef3-0a59dba96193 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-06 08:21:28.032207553 +0800 CST StartTs:457152628763131907 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003b12d80 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457152630100066306} {CheckpointTs:457152674782511112 MinTableBarrierTs:457152675831087109 AdminJobType:noop} span: {tableID:112, startKey:7480000000000000ff705f720000000000fa, endKey:7480000000000000ff705f730000000000fa}, resolvedTs: 457152674782511112, checkpointTs: 457152674782511112, state: Preparing span: {tableID:116, startKey:7480000000000000ff745f720000000000fa, endKey:7480000000000000ff745f730000000000fa}, resolvedTs: 457152674782511112, checkpointTs: 457152674782511112, state: Preparing span: {tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}, resolvedTs: 457152674782511112, checkpointTs: 457152674782511112, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/97d46017-30ff-4beb-994a-38d0f5e943e4 {"id":"97d46017-30ff-4beb-994a-38d0f5e943e4","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899060} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 2 /tidb/cdc/default/__cdc_meta__/owner/2231960879054d4e 97d46017-30ff-4beb-994a-38d0f5e943e4 /tidb/cdc/default/default/changefeed/info/e684662d-022f-4a79-9ef3-0a59dba96193 {"upstream-id":7489988628380945915,"namespace":"default","changefeed-id":"e684662d-022f-4a79-9ef3-0a59dba96193","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-06T08:21:28.032207553+08:00","start-ts":457152628763131907,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457152630100066306} /tidb/cdc/default/default/changefeed/status/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":457152674782511112,"min-table-barrier-ts":457152676092968965,"admin-job-type":0} /tidb/cdc/default/default/task/position/97d46017-30ff-4beb-994a-38d0f5e943e4/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489988628380945915 {"id":7489988628380945915,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x cdc.test cli capture list 2>&1 | grep '"is-owner": true' ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table batch_add_table.finish_mark not exists for 18-th check, retry later 0 0 0 0 0 0 0 0 --:--:-- 0:00:10 --:--:-- 0 wait process cdc.test exit for 1-th time... Run finished, takes 2.202233794s INSERT - Takes(s): 2.2, Count: 5000, OPS: 2279.7, Avg(us): 4010, Min(us): 1202, Max(us): 68305, 95th(us): 4000, 99th(us): 61000 "is-owner": true, run task successfully wait process cdc.test exit for 2-th time... 0 0 0 0 0 0 0 0 --:--:-- 0:00:11 --:--:-- 0{"level":"warn","ts":"2025-04-06T08:24:25.124241+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001cdd340/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2025-04-06T08:24:25.125006+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000985a40/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} 0 0 0 0 0 0 0 0 --:--:-- 0:00:12 --:--:-- 0{"level":"warn","ts":"2025-04-06T08:24:25.217232+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc002085500/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489989421666590001 TiDB Cluster ID. tikv_gc_leader_uuid 65821efe6540010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-x-pbmg2, pid:7342, start at 2025-04-06 08:24:24.747859185 +0800 CST m=+3.170556320 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:26:24.758 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:24:24.759 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:14:24.759 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489989421666590001 TiDB Cluster ID. tikv_gc_leader_uuid 65821efe6540010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-x-pbmg2, pid:7342, start at 2025-04-06 08:24:24.747859185 +0800 CST m=+3.170556320 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:26:24.758 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:24:24.759 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:14:24.759 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489989426892576195 TiDB Cluster ID. tikv_gc_leader_uuid 65821efe7e40004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-x-pbmg2, pid:7418, start at 2025-04-06 08:24:24.833566392 +0800 CST m=+3.182657473 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:26:24.845 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:24:24.825 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:14:24.825 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Sun Apr 6 08:24:25 CST 2025] <<<<<< run test case changefeed_resume_with_checkpoint_ts success! >>>>>> Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/kill_owner_with_ddl/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/kill_owner_with_ddl/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/kill_owner_with_ddl/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/kill_owner_with_ddl/tiflash/log/proxy.log"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/kill_owner_with_ddl/tiflash-proxy.toml"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table batch_add_table.finish_mark not exists for 19-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 06 Apr 2025 00:24:25 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/57e8a4c9-375c-4651-9417-f034bd4ad18f {"id":"57e8a4c9-375c-4651-9417-f034bd4ad18f","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899063} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087bb947ff 57e8a4c9-375c-4651-9417-f034bd4ad18f /tidb/cdc/default/default/upstream/7489989388887227743 {"id":7489989388887227743,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/57e8a4c9-375c-4651-9417-f034bd4ad18f {"id":"57e8a4c9-375c-4651-9417-f034bd4ad18f","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899063} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087bb947ff 57e8a4c9-375c-4651-9417-f034bd4ad18f /tidb/cdc/default/default/upstream/7489989388887227743 {"id":7489989388887227743,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/57e8a4c9-375c-4651-9417-f034bd4ad18f {"id":"57e8a4c9-375c-4651-9417-f034bd4ad18f","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899063} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087bb947ff 57e8a4c9-375c-4651-9417-f034bd4ad18f /tidb/cdc/default/default/upstream/7489989388887227743 {"id":7489989388887227743,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x owner pid: 5512 owner id 97d46017-30ff-4beb-994a-38d0f5e943e4 [Sun Apr 6 08:24:26 CST 2025] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.56245626.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_resign_owner.server2.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_resign_owner.server2 --cluster-id default --addr 127.0.0.1:8301 ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table consistent_replicate_storage_file.usertable exists table consistent_replicate_storage_file.check1 not exists for 1-th check, retry later 0 0 0 0 0 0 0 0 --:--:-- 0:00:13 --:--:-- 0{"level":"warn","ts":"2025-04-06T08:24:27.109216+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000985a40/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":"2025-04-06T08:24:27.109281+0800","logger":"etcd-client","caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":"2025-04-06T08:24:27.109568+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001cdd340/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":"2025-04-06T08:24:27.109617+0800","logger":"etcd-client","caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} lease 223196087bb947ff revoked {"level":"warn","ts":"2025-04-06T08:24:27.202549+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc002085500/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} {"level":"info","ts":"2025-04-06T08:24:27.202618+0800","logger":"etcd-client","caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... table capture_session_done_during_task.t exists table batch_add_table.finish_mark exists table test.t_1 does not exists Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 1-th time, retry later 0 0 0 0 0 0 0 0 --:--:-- 0:00:14 --:--:-- 0 check diff successfully [Sun Apr 6 08:24:28 CST 2025] <<<<<< START cdc server in kill_owner_with_ddl case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kill_owner_with_ddl.87108712.out server --log-file /tmp/tidb_cdc_test/kill_owner_with_ddl/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/kill_owner_with_ddl/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2025/04/06 08:24:28.480 +08:00] [INFO] [store_cache.go:532] ["change store resolve state"] [store=2] [addr=127.0.0.1:20162] [from=unresolved] [to=resolved] [liveness-state=reachable] [2025/04/06 08:24:28.480 +08:00] [INFO] [store_cache.go:532] ["change store resolve state"] [store=3] [addr=127.0.0.1:20160] [from=unresolved] [to=resolved] [liveness-state=reachable] [2025/04/06 08:24:28.480 +08:00] [INFO] [store_cache.go:532] ["change store resolve state"] [store=142] [addr=127.0.0.1:9500] [from=unresolved] [to=resolved] [liveness-state=reachable] [2025/04/06 08:24:28.499 +08:00] [INFO] [main.go:196] ["genLock done"] [2025/04/06 08:24:28.500 +08:00] [INFO] [tso_client.go:147] ["[tso] exit tso dispatcher check loop"] [2025/04/06 08:24:28.500 +08:00] [INFO] [pd_service_discovery.go:551] ["[pd] exit member loop due to context canceled"] [2025/04/06 08:24:28.500 +08:00] [INFO] [tso_dispatcher.go:490] ["[tso] exit tso connection contexts updater"] [dc-location=global] [2025/04/06 08:24:28.500 +08:00] [INFO] [tso_dispatcher.go:264] ["[tso] stop fetching the pending tso requests due to context canceled"] [dc-location=global] [2025/04/06 08:24:28.500 +08:00] [INFO] [tso_dispatcher.go:158] ["[tso] exit tso deadline watcher"] [dc-location=global] [2025/04/06 08:24:28.500 +08:00] [INFO] [resource_manager_client.go:296] ["[resource manager] exit resource token dispatcher"] [2025/04/06 08:24:28.500 +08:00] [INFO] [tso_dispatcher.go:201] ["[tso] exit tso dispatcher"] [dc-location=global] [2025/04/06 08:24:28.500 +08:00] [INFO] [tso_client.go:157] ["[tso] closing tso client"] [2025/04/06 08:24:28.500 +08:00] [INFO] [tso_client.go:162] ["[tso] close tso client"] [2025/04/06 08:24:28.500 +08:00] [INFO] [tso_stream.go:359] ["tsoStream.recvLoop ended"] [stream=127.0.0.1:2379-1] [error="rpc error: code = Canceled desc = context canceled"] [errorVerbose="rpc error: code = Canceled desc = context canceled\ngithub.com/tikv/pd/client.(*tsoStream).recvLoop\n\t/go/pkg/mod/github.com/tikv/pd/client@v0.0.0-20241111073742-238d4d79ea31/tso_stream.go:427\nruntime.goexit\n\t/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.23.4.linux-amd64/src/runtime/asm_amd64.s:1700"] [2025/04/06 08:24:28.500 +08:00] [INFO] [tso_client.go:164] ["[tso] tso client is closed"] [2025/04/06 08:24:28.500 +08:00] [INFO] [pd_service_discovery.go:644] ["[pd] close pd service discovery client"] wait process cdc.test exit for 1-th time... table consistent_replicate_storage_file.check1 not exists for 2-th check, retry later 0 0 0 0 0 0 0 0 --:--:-- 0:00:15 --:--:-- 0 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/cli_with_auth/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... wait process cdc.test exit for 2-th time... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 06 Apr 2025 00:24:29 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/e684662d-022f-4a79-9ef3-0a59dba96193 {UpstreamID:7489988628380945915 Namespace:default ID:e684662d-022f-4a79-9ef3-0a59dba96193 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-06 08:21:28.032207553 +0800 CST StartTs:457152628763131907 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0043be1b0 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457152630100066306} {CheckpointTs:457152677403951113 MinTableBarrierTs:457152677403951113 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/97d46017-30ff-4beb-994a-38d0f5e943e4 {"id":"97d46017-30ff-4beb-994a-38d0f5e943e4","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899060} /tidb/cdc/default/__cdc_meta__/capture/de3ddb64-d966-49b3-94fc-587b4a667cfa {"id":"de3ddb64-d966-49b3-94fc-587b4a667cfa","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899066} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 2 /tidb/cdc/default/__cdc_meta__/owner/2231960879054d4e 97d46017-30ff-4beb-994a-38d0f5e943e4 /tidb/cdc/default/__cdc_meta__/owner/2231960879054d99 de3ddb64-d966-49b3-94fc-587b4a667cfa /tidb/cdc/default/default/changefeed/info/e684662d-022f-4a79-9ef3-0a59dba96193 {"upstream-id":7489988628380945915,"namespace":"default","changefeed-id":"e684662d-022f-4a79-9ef3-0a59dba96193","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-06T08:21:28.032207553+08:00","start-ts":457152628763131907,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457152630100066306} /tidb/cdc/default/default/changefeed/status/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":457152677403951113,"min-table-barrier-ts":457152677403951113,"admin-job-type":0} /tidb/cdc/default/default/task/position/97d46017-30ff-4beb-994a-38d0f5e943e4/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/de3ddb64-d966-49b3-94fc-587b4a667cfa/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489988628380945915 {"id":7489988628380945915,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/e684662d-022f-4a79-9ef3-0a59dba96193 {UpstreamID:7489988628380945915 Namespace:default ID:e684662d-022f-4a79-9ef3-0a59dba96193 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-06 08:21:28.032207553 +0800 CST StartTs:457152628763131907 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0043be1b0 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457152630100066306} {CheckpointTs:457152677403951113 MinTableBarrierTs:457152677403951113 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/97d46017-30ff-4beb-994a-38d0f5e943e4 {"id":"97d46017-30ff-4beb-994a-38d0f5e943e4","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899060} /tidb/cdc/default/__cdc_meta__/capture/de3ddb64-d966-49b3-94fc-587b4a667cfa {"id":"de3ddb64-d966-49b3-94fc-587b4a667cfa","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899066} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 2 /tidb/cdc/default/__cdc_meta__/owner/2231960879054d4e 97d46017-30ff-4beb-994a-38d0f5e943e4 /tidb/cdc/default/__cdc_meta__/owner/2231960879054d99 de3ddb64-d966-49b3-94fc-587b4a667cfa /tidb/cdc/default/default/changefeed/info/e684662d-022f-4a79-9ef3-0a59dba96193 {"upstream-id":7489988628380945915,"namespace":"default","changefeed-id":"e684662d-022f-4a79-9ef3-0a59dba96193","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-06T08:21:28.032207553+08:00","start-ts":457152628763131907,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457152630100066306} /tidb/cdc/default/default/changefeed/status/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":457152677403951113,"min-table-barrier-ts":457152677403951113,"admin-job-type":0} /tidb/cdc/default/default/task/position/97d46017-30ff-4beb-994a+ grep -q 'failed to get info:' -38d0f5e943e4/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/de3ddb64-d966-49b3-94fc-587b4a667cfa/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489988628380945915 {"id":7489988628380945915,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/e684662d-022f-4a79-9ef3-0a59dba96193 {UpstreamID:7489988628380945915 Namespace:default ID:e684662d-022f-4a79-9ef3-0a59dba96193 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-06 08:21:28.032207553 +0800 CST StartTs:457152628763131907 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0043be1b0 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457152630100066306} {CheckpointTs:457152677403951113 MinTableBarrierTs:457152677403951113 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/97d46017-30ff-4beb-994a-38d0f5e943e4 {"id":"97d46017-30ff-4beb-994a-38d0f5e943e4","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899060} /tidb/cdc/default/__cdc_meta__/capture/de3ddb64-d966-49b3-94fc-587b4a667cfa {"id":"de3ddb64-d966-49b3-94fc-587b4a667cfa","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899066} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 2 /tidb/cdc/default/__cdc_meta__/owner/2231960879054d4e 97d46017-30ff-4beb-994a-38d0f5e943e4 /tidb/cdc/default/__cdc_meta__/owner/2231960879054d99 de3ddb64-d966-49b3-94fc-587b4a667cfa /tidb/cdc/default/default/changefeed/info/e684662d-022f-4a79-9ef3-0a59dba96193 {"upstream-id":7489988628380945915,"namespace":"default","changefeed-id":"e684662d-022f-4a79-9ef3-0a59dba96193","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-06T08:21:28.032207553+08:00","start-ts":457152628763131907,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457152630100066306} /tidb/cdc/default/default/changefeed/status/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":457152677403951113,"min-table-barrier-ts":457152677403951113,"admin-job-type":0} /tidb/cdc/default/default/task/position/97d46017-30ff-4beb-994a+ grep -q 'etcd info' -38d0f5e943e4/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/de3ddb64-d966-49b3-94fc-587b4a667cfa/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489988628380945915 {"id":7489988628380945915,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list 2>&1 | grep -v "97d46017-30ff-4beb-994a-38d0f5e943e4" | grep id ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Sun Apr 6 08:24:29 CST 2025] <<<<<< run test case batch_add_table success! >>>>>> 0 0 0 0 0 0 0 0 --:--:-- 0:00:16 --:--:-- 0 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 2-th time, retry later "cluster-id": "default" "id": "de3ddb64-d966-49b3-94fc-587b4a667cfa", "cluster-id": "default" run task successfully table consistent_replicate_storage_file.check1 not exists for 3-th check, retry later 0 0 0 0 0 0 0 0 --:--:-- 0:00:17 --:--:-- 0{"level":"warn","ts":"2025-04-06T08:24:31.125885+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001cdd340/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2025-04-06T08:24:31.127035+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000985a40/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} 0 0 0 0 0 0 0 0 --:--:-- 0:00:18 --:--:-- 0{"level":"warn","ts":"2025-04-06T08:24:31.218561+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc002085500/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 06 Apr 2025 00:24:31 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/048bc400-983f-47f7-bc21-6ea57b623b4c {"id":"048bc400-983f-47f7-bc21-6ea57b623b4c","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899068} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087bdb37d6 048bc400-983f-47f7-bc21-6ea57b623b4c /tidb/cdc/default/default/upstream/7489989421666590001 {"id":7489989421666590001,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/048bc400-983f-47f7-bc21-6ea57b623b4c {"id":"048bc400-983f-47f7-bc21-6ea57b623b4c","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899068} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087bdb37d6 048bc400-983f-47f7-bc21-6ea57b623b4c /tidb/cdc/default/default/upstream/7489989421666590001 {"id":7489989421666590001,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/048bc400-983f-47f7-bc21-6ea57b623b4c {"id":"048bc400-983f-47f7-bc21-6ea57b623b4c","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899068} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087bdb37d6 048bc400-983f-47f7-bc21-6ea57b623b4c /tidb/cdc/default/default/upstream/7489989421666590001 {"id":7489989421666590001,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: 9d7077c4-d482-42fd-b85e-bd9b18f0d9ec Info: {"upstream_id":7489989421666590001,"namespace":"default","id":"9d7077c4-d482-42fd-b85e-bd9b18f0d9ec","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create_time":"2025-04-06T08:24:31.34006381+08:00","start_ts":457152678123012101,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457152678123012101,"checkpoint_ts":457152678123012101,"checkpoint_time":"2025-04-06 08:24:31.209"} table kill_owner_with_ddl.t1 not exists for 1-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489989450251631464 TiDB Cluster ID. tikv_gc_leader_uuid 65821f0389c000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-2-dwpgc, pid:7398, start at 2025-04-06 08:24:30.024651026 +0800 CST m=+2.339804942 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:26:30.032 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:24:29.991 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:14:29.991 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489989450251631464 TiDB Cluster ID. tikv_gc_leader_uuid 65821f0389c000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-2-dwpgc, pid:7398, start at 2025-04-06 08:24:30.024651026 +0800 CST m=+2.339804942 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:26:30.032 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:24:29.991 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:14:29.991 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489989454743518542 TiDB Cluster ID. tikv_gc_leader_uuid 65821f03944000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-2-dwpgc, pid:7459, start at 2025-04-06 08:24:30.082037958 +0800 CST m=+2.337975416 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:26:30.090 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:24:30.083 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:14:30.083 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/sink_hang/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/sink_hang/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/sink_hang/tiflash/db/proxy"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/sink_hang/tiflash/log/proxy.log"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/sink_hang/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } capture_id: de3ddb64-d966-49b3-94fc-587b4a667cfa % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 2 100 2 0 0 490 0 --:--:-- --:--:-- --:--:-- 666 {}cdc.test cli capture list --server 'http://127.0.0.1:8301' 2>&1 |grep de3ddb64-d966-49b3-94fc-587b4a667cfa -A1 | grep '"is-owner": true' start tidb cluster in /tmp/tidb_cdc_test/cli_with_auth Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489989454190546122 TiDB Cluster ID. tikv_gc_leader_uuid 65821f05344000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-z-4sl2f, pid:6743, start at 2025-04-06 08:24:31.737508336 +0800 CST m=+3.582708939 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:26:31.746 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:24:31.746 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:14:31.746 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489989454190546122 TiDB Cluster ID. tikv_gc_leader_uuid 65821f05344000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-z-4sl2f, pid:6743, start at 2025-04-06 08:24:31.737508336 +0800 CST m=+3.582708939 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:26:31.746 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:24:31.746 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:14:31.746 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489989453946105348 TiDB Cluster ID. tikv_gc_leader_uuid 65821f05590000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-z-4sl2f, pid:6802, start at 2025-04-06 08:24:31.860353694 +0800 CST m=+3.621820037 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:26:31.869 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:24:31.844 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:14:31.844 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/ddl_reentrant/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/ddl_reentrant/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/ddl_reentrant/tiflash/log/proxy.log"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/ddl_reentrant/tiflash/db/proxy"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/ddl_reentrant/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } check diff successfully check diff failed 1-th time, retry later table consistent_replicate_storage_file.check1 not exists for 4-th check, retry later 0 0 0 0 0 0 0 0 --:--:-- 0:00:19 --:--:-- 0 + run_sql 'SELECT * from test.t1' 127.0.0.1 3306 + check_contains 'b: 123456' + check_contains 'b: aaaaa' + check_contains 'c: ab\\\\c' + check_contains 'c: ab\\c' + stop_tidb_cluster table kill_owner_with_ddl.t1 exists [Sun Apr 6 08:24:34 CST 2025] <<<<<< START cdc server in sink_hang case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sink/dmlsink/txn/mysql/MySQLSinkExecDMLError=2*return(true)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sink_hang.88448846.out server --log-file /tmp/tidb_cdc_test/sink_hang/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/sink_hang/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 wait process 8715 exit for 1-th time... "is-owner": true, run task successfully test_resign_owner: pass wait process cdc.test exit for 1-th time... skip some DDLs in tidb v4.0.x [Sun Apr 6 08:24:34 CST 2025] <<<<<< START cdc server in ddl_reentrant case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_reentrant.80918093.out server --log-file /tmp/tidb_cdc_test/ddl_reentrant/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_reentrant/cdc_data --cluster-id default + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 0 0 0 0 0 0 0 0 --:--:-- 0:00:20 --:--:-- 0 wait process 8715 exit for 2-th time... 0 0 0 0 0 0 0 0 --:--:-- 0:00:21 --:--:-- 0{"level":"warn","ts":1743899074.7854717,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc004c516c0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":1743899074.7855241,"caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":1743899074.8122215,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc002eb6380/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":1743899074.8122756,"caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":1743899074.82178,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0053fd880/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} {"level":"info","ts":1743899074.8218215,"caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} table consistent_replicate_storage_file.check1 not exists for 5-th check, retry later wait process cdc.test exit for 2-th time... check diff successfully wait process cdc.test exit for 1-th time... table test.t2 not exists for 1-th check, retry later wait process 8715 exit for 3-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit run test case test_kill_capture [Sun Apr 6 08:24:35 CST 2025] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.57945796.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_kill_capture.server1.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_kill_capture.server1 --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (8715) - No such process wait process 8715 exit for 4-th time... process 8715 already exit check_capture_count http://127.0.0.1:2379 0 wait process cdc.test exit for 2-th time... parse error: Invalid numeric literal at line 1, column 6 run task successfully [Sun Apr 6 08:24:35 CST 2025] <<<<<< START cdc server in kill_owner_with_ddl case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sink/ddlsink/mysql/MySQLSinkExecDDLDelay=return(true);github.com/pingcap/tiflow/cdc/capture/ownerFlushIntervalInject=return(10)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kill_owner_with_ddl.88898891.out server --log-file /tmp/tidb_cdc_test/kill_owner_with_ddl/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/kill_owner_with_ddl/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Sun Apr 6 08:24:36 CST 2025] <<<<<< run test case capture_session_done_during_task success! >>>>>> 0 0 0 0 0 0 0 0 --:--:-- 0:00:22 --:--:-- 0 ____________________________________ *************************** 1. row *************************** primary_ts: 457152622755840000 secondary_ts: 457152624780640270 *************************** 2. row *************************** primary_ts: 457152630620160000 secondary_ts: 457152630849273857 *************************** 3. row *************************** primary_ts: 457152638484480000 secondary_ts: 457152638988845057 *************************** 4. row *************************** primary_ts: 457152646348800000 secondary_ts: 457152646577913857 *************************** 5. row *************************** primary_ts: 457152654213120000 secondary_ts: 457152654442233857 *************************** 6. row *************************** primary_ts: 457152662077440000 secondary_ts: 457152662332768257 *************************** 7. row *************************** primary_ts: 457152669941760000 secondary_ts: 457152670197088257 skip invalid syncpoint primary_ts: 457152622755840000, first_ddl_ts: 457152652360286213 skip invalid syncpoint primary_ts: 457152630620160000, first_ddl_ts: 457152652360286213 skip invalid syncpoint primary_ts: 457152638484480000, first_ddl_ts: 457152652360286213 skip invalid syncpoint primary_ts: 457152646348800000, first_ddl_ts: 457152652360286213 check diff successfully check diff successfully check diff successfully check diff successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Sun Apr 6 08:24:32 CST 2025] <<<<<< run test case syncpoint success! >>>>>> table test.t2 not exists for 2-th check, retry later 0 0 0 0 0 0 0 0 --:--:-- 0:00:23 --:--:-- 0{"level":"warn","ts":"2025-04-06T08:24:37.126973+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001cdd340/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2025-04-06T08:24:37.128285+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000985a40/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} 0 0 0 0 0 0 0 0 --:--:-- 0:00:24 --:--:-- 0{"level":"warn","ts":"2025-04-06T08:24:37.219964+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc002085500/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} table consistent_replicate_storage_file.check1 not exists for 6-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 06 Apr 2025 00:24:37 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6f79b4f5-9d0d-4e82-bae2-a5a919c0ae85 {"id":"6f79b4f5-9d0d-4e82-bae2-a5a919c0ae85","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899074} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087bf32cd8 6f79b4f5-9d0d-4e82-bae2-a5a919c0ae85 /tidb/cdc/default/default/upstream/7489989450251631464 {"id":7489989450251631464,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6f79b4f5-9d0d-4e82-bae2-a5a919c0ae85 {"id":"6f79b4f5-9d0d-4e82-bae2-a5a919c0ae85","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899074} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087bf32cd8 6f79b4f5-9d0d-4e82-bae2-a5a919c0ae85 /tidb/cdc/default/default/upstream/7489989450251631464 {"id":7489989450251631464,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6f79b4f5-9d0d-4e82-bae2-a5a919c0ae85 {"id":"6f79b4f5-9d0d-4e82-bae2-a5a919c0ae85","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899074} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087bf32cd8 6f79b4f5-9d0d-4e82-bae2-a5a919c0ae85 /tidb/cdc/default/default/upstream/7489989450251631464 {"id":7489989450251631464,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 06 Apr 2025 00:24:37 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/af48e955-9450-42b9-bd11-d34e3baf0a40 {"id":"af48e955-9450-42b9-bd11-d34e3baf0a40","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899074} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087bf4f0d4 af48e955-9450-42b9-bd11-d34e3baf0a40 /tidb/cdc/default/default/upstream/7489989454190546122 {"id":7489989454190546122,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/af48e955-9450-42b9-bd11-d34e3baf0a40 {"id":"af48e955-9450-42b9-bd11-d34e3baf0a40","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899074} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087bf4f0d4 af48e955-9450-42b9-bd11-d34e3baf0a40 /tidb/cdc/default/default/upstream/7489989454190546122 {"id":7489989454190546122,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/af48e955-9450-42b9-bd11-d34e3baf0a40 {"id":"af48e955-9450-42b9-bd11-d34e3baf0a40","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899074} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087bf4f0d4 af48e955-9450-42b9-bd11-d34e3baf0a40 /tidb/cdc/default/default/upstream/7489989454190546122 {"id":7489989454190546122,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [2025/04/06 08:24:35.253 +08:00] [ERROR] [job.go:75] ["Error 1062 (23000): Duplicate entry '08:34:55' for key 'ntest.d'"] [stack="github.com/pingcap/tiflow/tests/integration_tests/cdc/dailytest.execSqls\n\t/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/cdc/dailytest/job.go:75\ngithub.com/pingcap/tiflow/tests/integration_tests/cdc/dailytest.doSqls\n\t/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/cdc/dailytest/job.go:63\ngithub.com/pingcap/tiflow/tests/integration_tests/cdc/dailytest.doJob\n\t/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/cdc/dailytest/job.go:90"] [2025/04/06 08:24:35.255 +08:00] [ERROR] [job.go:75] ["Error 1062 (23000): Duplicate entry '08:34:57' for key 'ntest.d'"] [stack="github.com/pingcap/tiflow/tests/integration_tests/cdc/dailytest.execSqls\n\t/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/cdc/dailytest/job.go:75\ngithub.com/pingcap/tiflow/tests/integration_tests/cdc/dailytest.doSqls\n\t/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/cdc/dailytest/job.go:63\ngithub.com/pingcap/tiflow/tests/integration_tests/cdc/dailytest.doJob\n\t/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/cdc/dailytest/job.go:90"] [2025/04/06 08:24:35.256 +08:00] [ERROR] [job.go:75] ["Error 1062 (23000): Duplicate entry '08:34:58' for key 'ntest.d'"] [stack="github.com/pingcap/tiflow/tests/integration_tests/cdc/dailytest.execSqls\n\t/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/cdc/dailytest/job.go:75\ngithub.com/pingcap/tiflow/tests/integration_tests/cdc/dailytest.doSqls\n\t/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/cdc/dailytest/job.go:63\ngithub.com/pingcap/tiflow/tests/integration_tests/cdc/dailytest.doJob\n\t/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/cdc/dailytest/job.go:90"] [2025/04/06 08:24:35.257 +08:00] [ERROR] [job.go:75] ["Error 1062 (23000): Duplicate entry '08:34:54' for key 'ntest.d'"] [stack="github.com/pingcap/tiflow/tests/integration_tests/cdc/dailytest.execSqls\n\t/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/cdc/dailytest/job.go:75\ngithub.com/pingcap/tiflow/tests/integration_tests/cdc/dailytest.doSqls\n\t/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/cdc/dailytest/job.go:63\ngithub.com/pingcap/tiflow/tests/integration_tests/cdc/dailytest.doJob\n\t/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/cdc/dailytest/job.go:90"] [2025/04/06 08:24:35.258 +08:00] [ERROR] [job.go:75] ["Error 1062 (23000): Duplicate entry '08:34:56' for key 'ntest.d'"] [stack="github.com/pingcap/tiflow/tests/integration_tests/cdc/dailytest.execSqls\n\t/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/cdc/dailytest/job.go:75\ngithub.com/pingcap/tiflow/tests/integration_tests/cdc/dailytest.doSqls\n\t/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/cdc/dailytest/job.go:63\ngithub.com/pingcap/tiflow/tests/integration_tests/cdc/dailytest.doJob\n\t/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/cdc/dailytest/job.go:90"] [2025/04/06 08:24:35.329 +08:00] [ERROR] [job.go:75] ["Error 1062 (23000): Duplicate entry '08:34:54' for key 'itest.d'"] [stack="github.com/pingcap/tiflow/tests/integration_tests/cdc/dailytest.execSqls\n\t/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/cdc/dailytest/job.go:75\ngithub.com/pingcap/tiflow/tests/integration_tests/cdc/dailytest.doSqls\n\t/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/cdc/dailytest/job.go:63\ngithub.com/pingcap/tiflow/tests/integration_tests/cdc/dailytest.doJob\n\t/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/cdc/dailytest/job.go:90"] [2025/04/06 08:24:35.335 +08:00] [ERROR] [job.go:75] ["Error 1062 (23000): Duplicate entry '08:34:55' for key 'itest.d'"] [stack="github.com/pingcap/tiflow/tests/integration_tests/cdc/dailytest.execSqls\n\t/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/cdc/dailytest/job.go:75\ngithub.com/pingcap/tiflow/tests/integration_tests/cdc/dailytest.doSqls\n\t/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/cdc/dailytest/job.go:63\ngithub.com/pingcap/tiflow/tests/integration_tests/cdc/dailytest.doJob\n\t/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/cdc/dailytest/job.go:90"] [2025/04/06 08:24:35.533 +08:00] [ERROR] [job.go:75] ["Error 1062 (23000): Duplicate entry '08:35:26' for key 'itest.d'"] [stack="github.com/pingcap/tiflow/tests/integration_tests/cdc/dailytest.execSqls\n\t/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/cdc/dailytest/job.go:75\ngithub.com/pingcap/tiflow/tests/integration_tests/cdc/dailytest.doSqls\n\t/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/cdc/dailytest/job.go:63\ngithub.com/pingcap/tiflow/tests/integration_tests/cdc/dailytest.doJob\n\t/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/cdc/dailytest/job.go:90"] [2025/04/06 08:24:35.544 +08:00] [ERROR] [job.go:75] ["Error 1062 (23000): Duplicate entry '08:34:56' for key 'ptest.d'"] [stack="github.com/pingcap/tiflow/tests/integration_tests/cdc/dailytest.execSqls\n\t/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/cdc/dailytest/job.go:75\ngithub.com/pingcap/tiflow/tests/integration_tests/cdc/dailytest.doSqls\n\t/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/cdc/dailytest/job.go:63\ngithub.com/pingcap/tiflow/tests/integration_tests/cdc/dailytest.doJob\n\t/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/cdc/dailytest/job.go:90"] [2025/04/06 08:24:35.544 +08:00] [ERROR] [job.go:75] ["Error 1062 (23000): Duplicate entry '08:34:45' for key 'ptest.d'"] [stack="github.com/pingcap/tiflow/tests/integration_tests/cdc/dailytest.execSqls\n\t/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/cdc/dailytest/job.go:75\ngithub.com/pingcap/tiflow/tests/integration_tests/cdc/dailytest.doSqls\n\t/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/cdc/dailytest/job.go:63\ngithub.com/pingcap/tiflow/tests/integration_tests/cdc/dailytest.doJob\n\t/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/cdc/dailytest/job.go:90"] [2025/04/06 08:24:35.546 +08:00] [ERROR] [job.go:75] ["Error 1062 (23000): Duplicate entry '08:34:57' for key 'ptest.d'"] [stack="github.com/pingcap/tiflow/tests/integration_tests/cdc/dailytest.execSqls\n\t/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/cdc/dailytest/job.go:75\ngithub.com/pingcap/tiflow/tests/integration_tests/cdc/dailytest.doSqls\n\t/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/cdc/dailytest/job.go:63\ngithub.com/pingcap/tiflow/tests/integration_tests/cdc/dailytest.doJob\n\t/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/cdc/dailytest/job.go:90"] [2025/04/06 08:24:35.548 +08:00] [ERROR] [job.go:75] ["Error 1062 (23000): Duplicate entry '08:34:58' for key 'ptest.d'"] [stack="github.com/pingcap/tiflow/tests/integration_tests/cdc/dailytest.execSqls\n\t/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/cdc/dailytest/job.go:75\ngithub.com/pingcap/tiflow/tests/integration_tests/cdc/dailytest.doSqls\n\t/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/cdc/dailytest/job.go:63\ngithub.com/pingcap/tiflow/tests/integration_tests/cdc/dailytest.doJob\n\t/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/cdc/dailytest/job.go:90"] [2025/04/06 08:24:35.629 +08:00] [ERROR] [job.go:75] ["Error 1213 (40001): Deadlock found when trying to get lock; try restarting transaction"] [stack="github.com/pingcap/tiflow/tests/integration_tests/cdc/dailytest.execSqls\n\t/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/cdc/dailytest/job.go:75\ngithub.com/pingcap/tiflow/tests/integration_tests/cdc/dailytest.doSqls\n\t/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/cdc/dailytest/job.go:63\ngithub.com/pingcap/tiflow/tests/integration_tests/cdc/dailytest.doJob\n\t/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/cdc/dailytest/job.go:90"] [2025/04/06 08:24:35.654 +08:00] [ERROR] [job.go:75] ["Error 1062 (23000): Duplicate entry '08:35:40' for key 'ptest.d'"] [stack="github.com/pingcap/tiflow/tests/integration_tests/cdc/dailytest.execSqls\n\t/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/cdc/dailytest/job.go:75\ngithub.com/pingcap/tiflow/tests/integration_tests/cdc/dailytest.doSqls\n\t/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/cdc/dailytest/job.go:63\ngithub.com/pingcap/tiflow/tests/integration_tests/cdc/dailytest.doJob\n\t/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/cdc/dailytest/job.go:90"] [2025/04/06 08:24:35.654 +08:00] [ERROR] [job.go:75] ["Error 1062 (23000): Duplicate entry '08:35:39' for key 'ptest.d'"] [stack="github.com/pingcap/tiflow/tests/integration_tests/cdc/dailytest.execSqls\n\t/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/cdc/dailytest/job.go:75\ngithub.com/pingcap/tiflow/tests/integration_tests/cdc/dailytest.doSqls\n\t/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/cdc/dailytest/job.go:63\ngithub.com/pingcap/tiflow/tests/integration_tests/cdc/dailytest.doJob\n\t/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/cdc/dailytest/job.go:90"] [2025/04/06 08:24:37.069 +08:00] [ERROR] [job.go:75] ["Error 1213 (40001): Deadlock found when trying to get lock; try restarting transaction"] [stack="github.com/pingcap/tiflow/tests/integration_tests/cdc/dailytest.execSqls\n\t/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/cdc/dailytest/job.go:75\ngithub.com/pingcap/tiflow/tests/integration_tests/cdc/dailytest.doSqls\n\t/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/cdc/dailytest/job.go:63\ngithub.com/pingcap/tiflow/tests/integration_tests/cdc/dailytest.doJob\n\t/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/cdc/dailytest/job.go:90"] [2025/04/06 08:24:37.150 +08:00] [ERROR] [job.go:75] ["Error 1213 (40001): Deadlock found when trying to get lock; try restarting transaction"] [stack="github.com/pingcap/tiflow/tests/integration_tests/cdc/dailytest.execSqls\n\t/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/cdc/dailytest/job.go:75\ngithub.com/pingcap/tiflow/tests/integration_tests/cdc/dailytest.doSqls\n\t/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/cdc/dailytest/job.go:63\ngithub.com/pingcap/tiflow/tests/integration_tests/cdc/dailytest.doJob\n\t/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/cdc/dailytest/job.go:90"] \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } [Pipeline] // timeout ------------------------------------------ test ddl create database ddl_reentrant, is_reentrant: false restored_sql: CREATE DATABASE `ddl_reentrant` [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-periodics_cdc_integration_mysql_test-234/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 06 Apr 2025 00:24:38 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/e684662d-022f-4a79-9ef3-0a59dba96193 {UpstreamID:7489988628380945915 Namespace:default ID:e684662d-022f-4a79-9ef3-0a59dba96193 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-06 08:21:28.032207553 +0800 CST StartTs:457152628763131907 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc004524240 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457152630100066306} {CheckpointTs:457152678727516161 MinTableBarrierTs:457152679776354305 AdminJobType:noop} span: {tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}, resolvedTs: 457152678727516161, checkpointTs: 457152678727516161, state: Preparing span: {tableID:112, startKey:7480000000000000ff705f720000000000fa, endKey:7480000000000000ff705f730000000000fa}, resolvedTs: 457152678727516161, checkpointTs: 457152678727516161, state: Preparing span: {tableID:116, startKey:7480000000000000ff745f720000000000fa, endKey:7480000000000000ff745f730000000000fa}, resolvedTs: 457152678727516161, checkpointTs: 457152678727516161, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f2f84621-e9c7-46bf-b856-8d4fb61c70e5 {"id":"f2f84621-e9c7-46bf-b856-8d4fb61c70e5","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899075} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 2 /tidb/cdc/default/__cdc_meta__/owner/2231960879054e16 f2f84621-e9c7-46bf-b856-8d4fb61c70e5 /tidb/cdc/default/default/changefeed/info/e684662d-022f-4a79-9ef3-0a59dba96193 {"upstream-id":7489988628380945915,"namespace":"default","changefeed-id":"e684662d-022f-4a79-9ef3-0a59dba96193","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-06T08:21:28.032207553+08:00","start-ts":457152628763131907,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457152630100066306} /tidb/cdc/default/default/changefeed/status/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":457152678727516161,"min-table-barrier-ts":457152679776354305,"admin-job-type":0} /tidb/cdc/default/default/task/position/f2f84621-e9c7-46bf-b856-8d4fb61c70e5/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489988628380945915 {"id":7489988628380945915,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/e684662d-022f-4a79-9ef3-0a59dba96193 {UpstreamID:7489988628380945915 Namespace:default ID:e684662d-022f-4a79-9ef3-0a59dba96193 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-06 08:21:28.032207553 +0800 CST StartTs:457152628763131907 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc004524240 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457152630100066306} {CheckpointTs:457152678727516161 MinTableBarrierTs:457152679776354305 AdminJobType:noop} span: {tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}, resolvedTs: 457152678727516161, checkpointTs: 457152678727516161, state: Preparing span: {tableID:112, startKey:7480000000000000ff705f720000000000fa, endKey:7480000000000000ff705f730000000000fa}, resolvedTs: 457152678727516161, checkpointTs: 457152678727516161, state: Preparing span: {tableID:116, startKey:7480000000000000ff745f720000000000fa, endKey:7480000000000000ff745f730000000000fa}, resolvedTs: 457152678727516161, checkpointTs: 457152678727516161, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f2f84621-e9c7-46bf-b856-8d4fb61c70e5 {"id":"f2f84621-e9c7-46bf-b856-8d4fb61c70e5","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899075} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 2 /tidb/cdc/default/__cdc_meta__/owner/2231960879054e16 f2f84621-e9c7-46bf-b856-8d4fb61c70e5 /tidb/cdc/default/default/changefeed/info/e684662d-022f-4a79-9ef3-0a59dba96193 check_ts_forward a2d89c91-f91a-41f7-8ca2-14e9437fcb8d {"upstream-id":7489988628380945915,"namespace":"default","changefeed-id":"e684662d-022f-4a79-9ef3-0a59dba96193","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-06T08:21:28.032207553+08:00","start-ts":457152628763131907,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457152630100066306} /tidb/cdc/default/default/changefeed/status/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":457152678727516161,"min-table-barrier-ts":457152679776354305,"admin-job-type":0} /tidb/cdc/default/default/task/position/f2f84621-e9c7-46bf-b856-8d4fb61c70e5/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489988628380945915 {"id":7489988628380945915,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/e684662d-022f-4a79-9ef3-0a59dba96193 {UpstreamID:7489988628380945915 Namespace:default ID:e684662d-022f-4a79-9ef3-0a59dba96193 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-06 08:21:28.032207553 +0800 CST StartTs:457152628763131907 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc004524240 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457152630100066306} {CheckpointTs:457152678727516161 MinTableBarrierTs:457152679776354305 AdminJobType:noop} span: {tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}, resolvedTs: 457152678727516161, checkpointTs: 457152678727516161, state: Preparing span: {tableID:112, startKey:7480000000000000ff705f720000000000fa, endKey:7480000000000000ff705f730000000000fa}, resolvedTs: 457152678727516161, checkpointTs: 457152678727516161, state: Preparing span: {tableID:116, startKey:7480000000000000ff745f720000000000fa, endKey:7480000000000000ff745f730000000000fa}, resolvedTs: 457152678727516161, checkpointTs: 457152678727516161, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f2f84621-e9c7-46bf-b856-8d4fb61c70e5 {"id":"f2f84621-e9c7-46bf-b856-8d4fb61c70e5","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899075} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 2 /tidb/cdc/default/__cdc_meta__/owner/2231960879054e16 f2f84621-e9c7-46bf-b856-8d4fb61c70e5 /tidb/cdc/default/default/changefeed/info/e684662d-022f-4a79-9ef3-0a59dba96193 {"upstream-id":7489988628380945915,"namespace":"default","changefeed-id":"e684662d-022f-4a79-9ef3-0a59dba96193","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-06T08:21:28.032207553+08:00","start-ts":457152628763131907,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457152630100066306} /tidb/cdc/default/default/changefeed/status/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":457152678727516161,"min-table-barrier-ts":457152679776354305,"admin-job-type":0} /tidb/cdc/default/default/task/position/f2f84621-e9c7-46bf-b856-8d4fb61c70e5/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489988628380945915 {"id":7489988628380945915,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x cdc.test cli capture list 2>&1 | grep '"is-owner": true' [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } table test.t2 not exists for 3-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 06 Apr 2025 00:24:38 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/9d7077c4-d482-42fd-b85e-bd9b18f0d9ec {UpstreamID:7489989421666590001 Namespace:default ID:9d7077c4-d482-42fd-b85e-bd9b18f0d9ec SinkURI:mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1 CreateTime:2025-04-06 08:24:31.34006381 +0800 CST StartTs:457152678123012101 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003d4a7e0 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457152678149226502} {CheckpointTs:457152679840055297 MinTableBarrierTs:457152679840055297 AdminJobType:noop} span: {tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}, resolvedTs: 457152679840055297, checkpointTs: 457152679840055297, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/94a89358-8f6c-4d68-99f1-80ed69185de2 {"id":"94a89358-8f6c-4d68-99f1-80ed69185de2","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899076} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087bdb3858 94a89358-8f6c-4d68-99f1-80ed69185de2 /tidb/cdc/default/default/changefeed/info/9d7077c4-d482-42fd-b85e-bd9b18f0d9ec {"upstream-id":7489989421666590001,"namespace":"default","changefeed-id":"9d7077c4-d482-42fd-b85e-bd9b18f0d9ec","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2025-04-06T08:24:31.34006381+08:00","start-ts":457152678123012101,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457152678149226502} /tidb/cdc/default/default/changefeed/status/9d7077c4-d482-42fd-b85e-bd9b18f0d9ec {"checkpoint-ts":457152679840055297,"min-table-barrier-ts":457152679840055297,"admin-job-type":0} /tidb/cdc/default/default/task/position/94a89358-8f6c-4d68-99f1-80ed69185de2/9d7077c4-d482-42fd-b85e-bd9b18f0d9ec {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489989421666590001 {"id":7489989421666590001,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/9d7077c4-d482-42fd-b85e-bd9b18f0d9ec {UpstreamID:7489989421666590001 Namespace:default ID:9d7077c4-d482-42fd-b85e-bd9b18f0d9ec SinkURI:mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1 CreateTime:2025-04-06 08:24:31.34006381 +0800 CST StartTs:457152678123012101 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003d4a7e0 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457152678149226502} {CheckpointTs:457152679840055297 MinTableBarrierTs:457152679840055297 AdminJobType:noop} span: {tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}, resolvedTs: 457152679840055297, checkpointTs: 457152679840055297, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/94a89358-8f6c-4d68-99f1-80ed69185de2 {"id":"94a89358-8f6c-4d68-99f1-80ed69185de2","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899076} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087bdb3858 94a89358-8f6c-4d68-99f1-80ed69185de2 /tidb/cdc/default/default/changefeed/info/9d7077c4-d482-42fd-b85e-bd9b18f0d9ec {"upstream-id":7489989421666590001,"namespace":"default","changefeed-id":"9d7077c4-d482-42fd-b85e-bd9b18f0d9ec","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2025-04-06T08:24:31.34006381+08:00","start-ts":457152678123012101,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457152678149226502} /tidb/cdc/default/default/changefeed/status/9d7077c4-d482-42fd-b85e-bd9b18f0d9ec {"checkpoint-ts":457152679840055297,"min-table-barrier-ts":457152679840055297,"admin-job-type":0} /tidb/cdc/default/default/task/position/94a89358-8f6c-4d68-99f1-80ed69185de2/9d7077c4-d482-42fd-b85e-bd9b18f0d9ec {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489989421666590001 {"id":7489989421666590001,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/9d7077c4-d482-42fd-b85e-bd9b18f0d9ec {UpstreamID:7489989421666590001 Namespace:default ID:9d7077c4-d482-42fd-b85e-bd9b18f0d9ec SinkURI:mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1 CreateTime:2025-04-06 08:24:31.34006381 +0800 CST StartTs:457152678123012101 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003d4a7e0 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457152678149226502} {CheckpointTs:457152679840055297 MinTableBarrierTs:457152679840055297 AdminJobType:noop} span: {tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}, resolvedTs: 457152679840055297, checkpointTs: 457152679840055297, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/94a89358-8f6c-4d68-99f1-80ed69185de2 {"id":"94a89358-8f6c-4d68-99f1-80ed69185de2","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899076} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087bdb3858 94a89358-8f6c-4d68-99f1-80ed69185de2 /tidb/cdc/default/default/changefeed/info/9d7077c4-d482-42fd-b85e-bd9b18f0d9ec {"upstream-id":7489989421666590001,"namespace":"default","changefeed-id":"9d7077c4-d482-42fd-b85e-bd9b18f0d9ec","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2025-04-06T08:24:31.34006381+08:00","start-ts":457152678123012101,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457152678149226502} /tidb/cdc/default/default/changefeed/status/9d7077c4-d482-42fd-b85e-bd9b18f0d9ec {"checkpoint-ts":457152679840055297,"min-table-barrier-ts":457152679840055297,"admin-job-type":0} /tidb/cdc/default/default/task/position/94a89358-8f6c-4d68-99f1-80ed69185de2/9d7077c4-d482-42fd-b85e-bd9b18f0d9ec {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489989421666590001 {"id":7489989421666590001,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x check_capture_count http://127.0.0.1:2379 1 run task successfully Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table consistent_replicate_storage_file.check1 not exists for 7-th check, retry later check_changefeed_status 127.0.0.1:8300 d4a8da3a-82fd-43e5-9108-7aa4dda87fc4 normal last_error null + endpoint=127.0.0.1:8300 + changefeed_id=d4a8da3a-82fd-43e5-9108-7aa4dda87fc4 + expected_state=normal + field=last_error + error_pattern=null ++ curl 127.0.0.1:8300/api/v2/changefeeds/d4a8da3a-82fd-43e5-9108-7aa4dda87fc4/status % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 86 100 86 0 0 662 0 --:--:-- --:--:-- --:--:-- 666 + info='{"state":"normal","resolved_ts":457152679730216962,"checkpoint_ts":457152679730216962}' + echo '{"state":"normal","resolved_ts":457152679730216962,"checkpoint_ts":457152679730216962}' {"state":"normal","resolved_ts":457152679730216962,"checkpoint_ts":457152679730216962} ++ echo '{"state":"normal","resolved_ts":457152679730216962,"checkpoint_ts":457152679730216962}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] + [[ -z last_error ]] ++ echo '{"state":"normal","resolved_ts":457152679730216962,"checkpoint_ts":457152679730216962}' ++ jq -r .last_error.message + error_msg=null + [[ ! null =~ null ]] run task successfully check_changefeed_status 127.0.0.1:8300 d4a8da3a-82fd-43e5-9108-7aa4dda87fc4 normal last_warning null + endpoint=127.0.0.1:8300 + changefeed_id=d4a8da3a-82fd-43e5-9108-7aa4dda87fc4 + expected_state=normal + field=last_warning + error_pattern=null ++ curl 127.0.0.1:8300/api/v2/changefeeds/d4a8da3a-82fd-43e5-9108-7aa4dda87fc4/status % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 86 100 86 0 0 1072 0 --:--:-- --:--:-- --:--:-- 1061 100 86 100 86 0 0 1071 0 --:--:-- --:--:-- --:--:-- 1061 + info='{"state":"normal","resolved_ts":457152679861288981,"checkpoint_ts":457152679861288981}' + echo '{"state":"normal","resolved_ts":457152679861288981,"checkpoint_ts":457152679861288981}' {"state":"normal","resolved_ts":457152679861288981,"checkpoint_ts":457152679861288981} ++ echo '{"state":"normal","resolved_ts":457152679861288981,"checkpoint_ts":457152679861288981}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] + [[ -z last_warning ]] ++ echo '{"state":"normal","resolved_ts":457152679861288981,"checkpoint_ts":457152679861288981}' ++ jq -r .last_warning.message + error_msg=null + [[ ! null =~ null ]] run task successfully table sink_hang.t1 exists table sink_hang.t2 not exists for 1-th check, retry later 0 0 0 0 0 0 0 0 --:--:-- 0:00:25 --:--:-- 0 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/multi_rocks/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... "is-owner": true, run task successfully Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) 0 0 0 0 0 0 0 0 --:--:-- 0:00:26 --:--:-- 0 table consistent_replicate_storage_file.check1 not exists for 8-th check, retry later changefeed is working normally rts: 457152679836647400->457152680216756200 checkpoint: 457152679836647400->457152680216756200 run task successfully check_ddl_executed /tmp/tidb_cdc_test/ddl_reentrant/cdc.log /tmp/tidb_cdc_test/ddl_reentrant/ddl_temp.sql true [2025/04/06 08:24:39.371 +08:00] [INFO] [mysql_ddl_sink.go:272] ["Exec DDL succeeded"] [namespace=default] [changefeed=a2d89c91-f91a-41f7-8ca2-14e9437fcb8d] [duration=29.565871ms] [sql="CREATE DATABASE `ddl_reentrant`"] run task successfully resume changefeed a2d89c91-f91a-41f7-8ca2-14e9437fcb8d from 457152679875969040 check_ts_forward a2d89c91-f91a-41f7-8ca2-14e9437fcb8d 0 0 0 0 0 0 0 0 --:--:-- 0:00:27 --:--:-- 0 table test.t2 not exists for 4-th check, retry later table sink_hang.t2 exists check diff failed 1-th time, retry later owner pid: 5799 owner id f2f84621-e9c7-46bf-b856-8d4fb61c70e5 table test.availability1 exists nonempty select id, val from test.availability1 where id=1 and val=1 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) 0 0 0 0 0 0 0 0 --:--:-- 0:00:28 --:--:-- 0 table consistent_replicate_storage_file.check1 not exists for 9-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/syncpoint_check_ts/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/multi_rocks Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... 0 0 0 0 0 0 0 0 --:--:-- 0:00:29 --:--:-- 0{"level":"warn","ts":"2025-04-06T08:24:43.128039+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001cdd340/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2025-04-06T08:24:43.130155+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000985a40/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} 100 135 100 135 0 0 4 0 0:00:33 0:00:30 0:00:03 27 100 135 100 135 0 0 4 0 0:00:33 0:00:30 0:00:03 33 + synced_status='{ "error_msg": "[CDC:ErrPDEtcdAPIError]etcd api call error: context deadline exceeded", "error_code": "CDC:ErrPDEtcdAPIError" }' ++ echo '{' '"error_msg":' '"[CDC:ErrPDEtcdAPIError]etcd' api call error: context deadline 'exceeded",' '"error_code":' '"CDC:ErrPDEtcdAPIError"' '}' ++ jq -r .error_code {"level":"warn","ts":"2025-04-06T08:24:43.221208+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc002085500/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} + error_code=CDC:ErrPDEtcdAPIError + cleanup_process cdc.test table test.t2 not exists for 5-th check, retry later changefeed is working normally rts: 457152679875969000->457152681016557600 checkpoint: 457152680754151400->457152681016557600 run task successfully check_ddl_executed /tmp/tidb_cdc_test/ddl_reentrant/cdc.log /tmp/tidb_cdc_test/ddl_reentrant/ddl_temp.sql false [2025/04/06 08:24:41.350 +08:00] [INFO] [mysql_ddl_sink.go:150] ["Execute DDL failed, but error can be ignored"] [startTs=457152679875969036] [ddl="CREATE DATABASE `ddl_reentrant`"] [namespace=default] [changefeed=a2d89c91-f91a-41f7-8ca2-14e9437fcb8d] [error="Error 1007 (HY000): Can't create database 'ddl_reentrant'; database exists"] run task successfully ------------------------------------------ test ddl create table ddl_reentrant.t3 (a int primary key, b int) partition by range(a) (partition p0 values less than (1000), partition p1 values less than (2000)), is_reentrant: false restored_sql: CREATE TABLE `ddl_reentrant`.`t3` (`a` INT PRIMARY KEY,`b` INT) PARTITION BY RANGE (`a`) (PARTITION `p0` VALUES LESS THAN (1000),PARTITION `p1` VALUES LESS THAN (2000)) check diff failed 2-th time, retry later nonempty select id, val from test.availability1 where id=1 and val=1 run task successfully [Sun Apr 6 08:24:43 CST 2025] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.59445946.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_kill_capture.server2.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_kill_capture.server2 --cluster-id default --addr 127.0.0.1:8301 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check_ts_forward a2d89c91-f91a-41f7-8ca2-14e9437fcb8d wait process cdc.test exit for 1-th time... [2025/04/06 08:24:43.264 +08:00] [WARN] [diff.go:557] ["checksum is not equal"] [table=`test`.`itest`] [where="((TRUE) AND TRUE)"] [sourceChecksum=1138534079] [targetChecksum=3014853762] [getSourceChecksumCost=6.267916ms] [getTargetChecksumCost=3.933534ms] [2025/04/06 08:24:43.339 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 0, b: 0, c: o, ahRhZ: VgauzvAfkJ, d: 08:24:34, }"] [row2="{ a: 0, b: 0, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:24:34, }"] [2025/04/06 08:24:43.344 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ d: 08:24:35, a: 1, b: 1, c: o, ahRhZ: VgauzvAfkJ, }"] [row2="{ ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:24:35, a: 1, b: 1, c: D, }"] [2025/04/06 08:24:43.445 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 2, b: 2, c: o, ahRhZ: VgauzvAfkJ, d: 08:28:00, }"] [row2="{ a: 2, b: 2, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:28:00, }"] [2025/04/06 08:24:43.545 +08:00] [WARN] [diff.go:937] ["find different row"] [column=ahRhZ] [row1="{ ahRhZ: VgauzvAfkJ, d: 08:24:37, a: 3, b: 3, c: o, }"] [row2="{ a: 3, b: 3, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:24:37, }"] [2025/04/06 08:24:43.645 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 4, b: 4, c: o, ahRhZ: VgauzvAfkJ, d: 08:24:38, }"] [row2="{ ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:24:38, a: 4, b: 4, c: D, }"] [2025/04/06 08:24:43.745 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 5, b: 5, c: o, ahRhZ: VgauzvAfkJ, d: 08:24:39, }"] [row2="{ b: 5, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:24:39, a: 5, }"] [2025/04/06 08:24:43.845 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ ahRhZ: VgauzvAfkJ, d: 08:24:40, a: 6, b: 6, c: o, }"] [row2="{ c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:24:40, a: 6, b: 6, }"] [2025/04/06 08:24:43.945 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ c: o, ahRhZ: VgauzvAfkJ, d: 08:24:43, a: 9, b: 9, }"] [row2="{ a: 9, b: 9, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:24:43, }"] [2025/04/06 08:24:44.046 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 11, b: 11, c: o, ahRhZ: VgauzvAfkJ, d: 08:24:45, }"] [row2="{ c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:24:45, a: 11, b: 11, }"] [2025/04/06 08:24:44.146 +08:00] [WARN] [diff.go:937] ["find different row"] [column=ahRhZ] [row1="{ a: 12, b: 12, c: o, ahRhZ: VgauzvAfkJ, d: 08:24:46, }"] [row2="{ b: 12, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:24:46, a: 12, }"] [2025/04/06 08:24:44.246 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ c: o, ahRhZ: VgauzvAfkJ, d: 08:24:47, a: 13, b: 13, }"] [row2="{ c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:24:47, a: 13, b: 13, }"] [2025/04/06 08:24:44.346 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ ahRhZ: VgauzvAfkJ, d: 08:24:48, a: 14, b: 14, c: o, }"] [row2="{ ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:24:48, a: 14, b: 14, c: D, }"] [2025/04/06 08:24:44.446 +08:00] [WARN] [diff.go:937] ["find different row"] [column=ahRhZ] [row1="{ a: 15, b: 15, c: o, ahRhZ: VgauzvAfkJ, d: 08:24:49, }"] [row2="{ d: 08:24:49, a: 15, b: 15, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, }"] wait process cdc.test exit for 2-th time... [2025/04/06 08:24:44.547 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ d: 08:24:50, a: 16, b: 16, c: o, ahRhZ: VgauzvAfkJ, }"] [row2="{ a: 16, b: 16, c: GMPpz, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:24:50, }"] [2025/04/06 08:24:44.647 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ b: 18, c: o, ahRhZ: VgauzvAfkJ, d: 08:32:50, a: 18, }"] [row2="{ a: 18, b: 18, c: D, ahRhZ: ERhmLPhT3Oxdr6me63iX6jtBjG, d: 08:32:50, }"] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489989501405550986 TiDB Cluster ID. tikv_gc_leader_uuid 65821f0fe40000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-c-g3td9, pid:10127, start at 2025-04-06 08:24:42.658004316 +0800 CST m=+2.584740279 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:26:42.666 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:24:42.640 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:14:42.640 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489989501405550986 TiDB Cluster ID. tikv_gc_leader_uuid 65821f0fe40000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-c-g3td9, pid:10127, start at 2025-04-06 08:24:42.658004316 +0800 CST m=+2.584740279 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:26:42.666 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:24:42.640 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:14:42.640 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489989487471029630 TiDB Cluster ID. tikv_gc_leader_uuid 65821f0fe080003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-c-g3td9, pid:10201, start at 2025-04-06 08:24:42.629766125 +0800 CST m=+2.488501752 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:26:42.638 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:24:42.637 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:14:42.637 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/cli_with_auth/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/cli_with_auth/tiflash/log/error.log arg matches is ArgMatches { args: {"log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/cli_with_auth/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/cli_with_auth/tiflash-proxy.toml"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/cli_with_auth/tiflash/db/proxy"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [2025/04/06 08:24:44.747 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ c: o, ahRhZ: VgauzvAfkJ, d: 08:27:45, a: 19, b: 19, }"] [row2="{ ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:27:45, a: 19, b: 19, c: D, }"] [2025/04/06 08:24:44.847 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 24, b: 24, c: o, ahRhZ: VgauzvAfkJ, d: 08:27:56, }"] [row2="{ b: 24, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:27:56, a: 24, }"] [2025/04/06 08:24:44.947 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ d: 08:25:00, a: 26, b: 26, c: o, ahRhZ: VgauzvAfkJ, }"] [row2="{ a: 26, b: 26, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:25:00, }"] cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit + stop_tidb_cluster [2025/04/06 08:24:45.048 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ ahRhZ: VgauzvAfkJ, d: 08:25:02, a: 28, b: 28, c: o, }"] [row2="{ d: 08:25:02, a: 28, b: 28, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, }"] [2025/04/06 08:24:45.148 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ c: o, ahRhZ: VgauzvAfkJ, d: 08:25:03, a: 29, b: 29, }"] [row2="{ d: 08:25:03, a: 29, b: 29, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, }"] table test.t2 not exists for 6-th check, retry later [2025/04/06 08:24:45.248 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ b: 30, c: o, ahRhZ: VgauzvAfkJ, d: 08:39:24, a: 30, }"] [row2="{ a: 30, b: 30, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:39:24, }"] [2025/04/06 08:24:45.348 +08:00] [WARN] [diff.go:937] ["find different row"] [column=ahRhZ] [row1="{ a: 31, b: 31, c: o, ahRhZ: VgauzvAfkJ, d: 08:25:05, }"] [row2="{ a: 31, b: 31, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:25:05, }"] [2025/04/06 08:24:45.448 +08:00] [WARN] [diff.go:937] ["find different row"] [column=ahRhZ] [row1="{ a: 33, b: 33, c: o, ahRhZ: VgauzvAfkJ, d: 08:25:07, }"] [row2="{ a: 33, b: 33, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:25:07, }"] table consistent_replicate_storage_file.check1 not exists for 10-th check, retry later wait process 8894 exit for 1-th time... /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (8894) - No such process wait process 8894 exit for 2-th time... process 8894 already exit check_capture_count http://127.0.0.1:2379 0 parse error: Invalid numeric literal at line 1, column 6 run task successfully [Sun Apr 6 08:24:45 CST 2025] <<<<<< START cdc server in kill_owner_with_ddl case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sink/ddlsink/mysql/MySQLSinkExecDDLDelay=return(true);github.com/pingcap/tiflow/cdc/capture/ownerFlushIntervalInject=return(10)' + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kill_owner_with_ddl.90659067.out server --log-file /tmp/tidb_cdc_test/kill_owner_with_ddl/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/kill_owner_with_ddl/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2025/04/06 08:24:45.549 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 34, b: 34, c: o, ahRhZ: VgauzvAfkJ, d: 08:25:08, }"] [row2="{ a: 34, b: 34, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:25:08, }"] [2025/04/06 08:24:45.649 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ c: o, ahRhZ: VgauzvAfkJ, d: 08:30:45, a: 36, b: 36, }"] [row2="{ d: 08:30:45, a: 36, b: 36, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, }"] run task failed 1-th time, retry later [2025/04/06 08:24:45.749 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 37, b: 37, c: o, ahRhZ: VgauzvAfkJ, d: 08:25:11, }"] [row2="{ b: 37, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:25:11, a: 37, }"] [2025/04/06 08:24:45.849 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 38, b: 38, c: o, ahRhZ: VgauzvAfkJ, d: 08:25:12, }"] [row2="{ ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:25:12, a: 38, b: 38, c: D, }"] [2025/04/06 08:24:45.949 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ d: 08:25:13, a: 39, b: 39, c: o, ahRhZ: VgauzvAfkJ, }"] [row2="{ ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:25:13, a: 39, b: 39, c: D, }"] check diff failed 3-th time, retry later [2025/04/06 08:24:46.049 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ c: o, ahRhZ: VgauzvAfkJ, d: 08:27:58, a: 40, b: 40, }"] [row2="{ ahRhZ: MztvMajZ6JfnUVFyCefJPV6WVDYEClNzMKzRXQ2, d: 08:27:58, a: 40, b: 40, c: D, }"] [2025/04/06 08:24:46.150 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 42, b: 42, c: o, ahRhZ: VgauzvAfkJ, d: 08:27:24, }"] [row2="{ a: 42, b: 42, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:27:24, }"] Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Debugger for raftstore-v2 is used Debugger for raftstore-v2 is used [2025/04/06 08:24:46.250 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 43, b: 43, c: o, ahRhZ: VgauzvAfkJ, d: 08:25:17, }"] [row2="{ ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:25:17, a: 43, b: 43, c: D, }"] [2025/04/06 08:24:46.350 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 44, b: 44, c: o, ahRhZ: VgauzvAfkJ, d: 08:25:18, }"] [row2="{ a: 44, b: 44, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:25:18, }"] [2025/04/06 08:24:46.450 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ b: 46, c: o, ahRhZ: VgauzvAfkJ, d: 08:25:20, a: 46, }"] [row2="{ a: 46, b: 46, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:25:20, }"] + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_with_auth.cli.11616.out cli tso query --pd=http://127.0.0.1:2379 [2025/04/06 08:24:46.551 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 48, b: 48, c: o, ahRhZ: VgauzvAfkJ, d: 08:25:22, }"] [row2="{ b: 48, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:25:22, a: 48, }"] [2025/04/06 08:24:46.651 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ ahRhZ: VgauzvAfkJ, d: 08:25:23, a: 49, b: 49, c: o, }"] [row2="{ b: 49, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:25:23, a: 49, }"] [2025/04/06 08:24:46.751 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 50, b: 50, c: o, ahRhZ: VgauzvAfkJ, d: 08:25:24, }"] [row2="{ d: 08:25:24, a: 50, b: 50, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, }"] [2025/04/06 08:24:46.851 +08:00] [WARN] [diff.go:937] ["find different row"] [column=ahRhZ] [row1="{ a: 51, b: 51, c: o, ahRhZ: VgauzvAfkJ, d: 08:25:25, }"] [row2="{ a: 51, b: 51, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:25:25, }"] [2025/04/06 08:24:46.951 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ d: 08:25:27, a: 53, b: 53, c: o, ahRhZ: VgauzvAfkJ, }"] [row2="{ a: 53, b: 53, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:25:27, }"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 06 Apr 2025 00:24:46 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/e684662d-022f-4a79-9ef3-0a59dba96193 {UpstreamID:7489988628380945915 Namespace:default ID:e684662d-022f-4a79-9ef3-0a59dba96193 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-06 08:21:28.032207553 +0800 CST StartTs:457152628763131907 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003fa4e10 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457152630100066306} {CheckpointTs:457152681873244164 MinTableBarrierTs:457152681873244164 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/07fd042a-1b40-44a1-8ed5-28784b309b86 {"id":"07fd042a-1b40-44a1-8ed5-28784b309b86","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899083} /tidb/cdc/default/__cdc_meta__/capture/f2f84621-e9c7-46bf-b856-8d4fb61c70e5 {"id":"f2f84621-e9c7-46bf-b856-8d4fb61c70e5","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899075} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 2 /tidb/cdc/default/__cdc_meta__/owner/2231960879054e16 f2f84621-e9c7-46bf-b856-8d4fb61c70e5 /tidb/cdc/default/__cdc_meta__/owner/2231960879054e78 07fd042a-1b40-44a1-8ed5-28784b309b86 /tidb/cdc/default/default/changefeed/info/e684662d-022f-4a79-9ef3-0a59dba96193 {"upstream-id":7489988628380945915,"namespace":"default","changefeed-id":"e684662d-022f-4a79-9ef3-0a59dba96193","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-06T08:21:28.032207553+08:00","start-ts":457152628763131907,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457152630100066306} /tidb/cdc/default/default/changefeed/status/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":457152681873244164,"min-table-barrier-ts":457152681873244164,"admin-job-type":0} /tidb/cdc/default/default/task/position/07fd042a-1b40-44a1-8ed5-28784b309b86/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/f2f84621-e9c7-46bf-b856-8d4fb61c70e5/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489988628380945915 {"id":7489988628380945915,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/e684662d-022f-4a79-9ef3-0a59dba96193 {UpstreamID:7489988628380945915 Namespace:default ID:e684662d-022f-4a79-9ef3-0a59dba96193 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-06 08:21:28.032207553 +0800 CST StartTs:457152628763131907 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003fa4e10 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457152630100066306} {CheckpointTs:457152681873244164 MinTableBarrierTs:457152681873244164 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/07fd042a-1b40-44a1-8ed5-28784b309b86 {"id":"07fd042a-1b40-44a1-8ed5-28784b309b86","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899083} /tidb/cdc/default/__cdc_meta__/capture/f2f84621-e9c7-46bf-b856-8d4fb61c70e5 {"id":"f2f84621-e9c7-46bf-b856-8d4fb61c70e5","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899075} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 2 /tidb/cdc/default/__cdc_meta__/owner/2231960879054e16 f2f84621-e9c7-46bf-b856-8d4fb61c70e5 /tidb/cdc/default/__cdc_meta__/owner/2231960879054e78 07fd042a-1b40-44a1-8ed5-28784b309b86 /tidb/cdc/default/default/changefeed/info/e684662d-022f-4a79-9ef3-0a59dba96193 {"upstream-id":7489988628380945915,"namespace":"default","changefeed-id":"e684662d-022f-4a79-9ef3-0a59dba96193","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-06T08:21:28.032207553+08:00","start-ts":457152628763131907,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457152630100066306} /tidb/cdc/default/default/changefeed/status/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":457152681873244164,"min-table-barrier-ts":457152681873244164,"admin-job-type":0} /tidb/cdc/default/default/task/position/07fd042a-1b40-44a1-8ed5+ grep -q 'failed to get info:' -28784b309b86/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/f2f84621-e9c7-46bf-b856-8d4fb61c70e5/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489988628380945915 {"id":7489988628380945915,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/e684662d-022f-4a79-9ef3-0a59dba96193 {UpstreamID:7489988628380945915 Namespace:default ID:e684662d-022f-4a79-9ef3-0a59dba96193 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-06 08:21:28.032207553 +0800 CST StartTs:457152628763131907 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003fa4e10 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457152630100066306} {CheckpointTs:457152681873244164 MinTableBarrierTs:457152681873244164 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/07fd042a-1b40-44a1-8ed5-28784b309b86 {"id":"07fd042a-1b40-44a1-8ed5-28784b309b86","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899083} /tidb/cdc/default/__cdc_meta__/capture/f2f84621-e9c7-46bf-b856-8d4fb61c70e5 {"id":"f2f84621-e9c7-46bf-b856-8d4fb61c70e5","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899075} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 2 /tidb/cdc/default/__cdc_meta__/owner/2231960879054e16 f2f84621-e9c7-46bf-b856-8d4fb61c70e5 /tidb/cdc/default/__cdc_meta__/owner/2231960879054e78 07fd042a-1b40-44a1-8ed5-28784b309b86 /tidb/cdc/default/default/changefeed/info/e684662d-022f-4a79-9ef3-0a59dba96193 {"upstream-id":7489988628380945915,"namespace":"default","changefeed-id":"e684662d-022f-4a79-9ef3-0a59dba96193","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-06T08:21:28.032207553+08:00","start-ts":457152628763131907,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457152630100066306} /tidb/cdc/default/default/changefeed/status/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":457152681873244164,"min-table-barrier-ts":457152681873244164,"admin-job-type":0} /tidb/cdc/default/default/task/position/07fd042a-1b40-44a1-8ed5+ grep -q 'etcd info' -28784b309b86/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/f2f84621-e9c7-46bf-b856-8d4fb61c70e5/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489988628380945915 {"id":7489988628380945915,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list 2>&1 | grep -v "f2f84621-e9c7-46bf-b856-8d4fb61c70e5" | grep id table test.t2 not exists for 7-th check, retry later [2025/04/06 08:24:47.051 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 54, b: 54, c: o, ahRhZ: VgauzvAfkJ, d: 08:25:28, }"] [row2="{ a: 54, b: 54, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:25:28, }"] [2025/04/06 08:24:47.152 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 55, b: 55, c: o, ahRhZ: VgauzvAfkJ, d: 08:25:29, }"] [row2="{ a: 55, b: 55, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:25:29, }"] start tidb cluster in /tmp/tidb_cdc_test/syncpoint_check_ts Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... table consistent_replicate_storage_file.check1 not exists for 11-th check, retry later [2025/04/06 08:24:47.252 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ b: 56, c: o, ahRhZ: VgauzvAfkJ, d: 08:27:57, a: 56, }"] [row2="{ a: 56, b: 56, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:27:57, }"] [2025/04/06 08:24:47.352 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ ahRhZ: VgauzvAfkJ, d: 08:25:31, a: 57, b: 57, c: o, }"] [row2="{ a: 57, b: 57, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:25:31, }"] [2025/04/06 08:24:47.452 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ d: 08:25:32, a: 58, b: 58, c: o, ahRhZ: VgauzvAfkJ, }"] [row2="{ c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:25:32, a: 58, b: 58, }"] check_ts_forward a2d89c91-f91a-41f7-8ca2-14e9437fcb8d [2025/04/06 08:24:47.553 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ b: 59, c: o, ahRhZ: VgauzvAfkJ, d: 08:25:33, a: 59, }"] [row2="{ d: 08:25:33, a: 59, b: 59, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, }"] [2025/04/06 08:24:47.653 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 60, b: 60, c: o, ahRhZ: VgauzvAfkJ, d: 08:25:34, }"] [row2="{ d: 08:25:34, a: 60, b: 60, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, }"] [2025/04/06 08:24:47.753 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 61, b: 61, c: o, ahRhZ: VgauzvAfkJ, d: 08:25:35, }"] [row2="{ c: D, ahRhZ: Tya4KiW5LBCEkk2fU4TDKd7dU8IwDeKx6JzngoMcTJlHa, d: 08:25:35, a: 61, b: 61, }"] [2025/04/06 08:24:47.853 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ d: 08:25:36, a: 62, b: 62, c: o, ahRhZ: VgauzvAfkJ, }"] [row2="{ b: 62, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:25:36, a: 62, }"] [2025/04/06 08:24:47.953 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ ahRhZ: VgauzvAfkJ, d: 08:25:37, a: 63, b: 63, c: o, }"] [row2="{ a: 63, b: 63, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:25:37, }"] check diff successfully [2025/04/06 08:24:48.054 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 65, b: 65, c: o, ahRhZ: VgauzvAfkJ, d: 08:25:39, }"] [row2="{ ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:25:39, a: 65, b: 65, c: D, }"] [2025/04/06 08:24:48.154 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 67, b: 67, c: o, ahRhZ: VgauzvAfkJ, d: 08:25:41, }"] [row2="{ b: 67, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:25:41, a: 67, }"] [2025/04/06 08:24:48.254 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 68, b: 68, c: o, ahRhZ: VgauzvAfkJ, d: 08:25:42, }"] [row2="{ ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:25:42, a: 68, b: 68, c: D, }"] + start_tidb_cluster --workdir /tmp/tidb_cdc_test/sql_mode The 1 times to try to start tidb cluster... wait process cdc.test exit for 1-th time... Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... "cluster-id": "default" "id": "07fd042a-1b40-44a1-8ed5-28784b309b86", "cluster-id": "default" run task successfully + set +x + tso='457152682220584961 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 457152682220584961 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/ddl_attributes/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [2025/04/06 08:24:48.354 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ b: 70, c: o, ahRhZ: VgauzvAfkJ, d: 08:25:44, a: 70, }"] [row2="{ a: 70, b: 70, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:25:44, }"] [2025/04/06 08:24:48.454 +08:00] [WARN] [diff.go:937] ["find different row"] [column=ahRhZ] [row1="{ a: 71, b: 71, c: o, ahRhZ: VgauzvAfkJ, d: 08:25:45, }"] [row2="{ a: 71, b: 71, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:25:45, }"] Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2025/04/06 08:24:48.555 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ ahRhZ: VgauzvAfkJ, d: 08:25:46, a: 72, b: 72, c: o, }"] [row2="{ a: 72, b: 72, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:25:46, }"] [2025/04/06 08:24:48.655 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ c: o, ahRhZ: VgauzvAfkJ, d: 08:25:47, a: 73, b: 73, }"] [row2="{ d: 08:25:47, a: 73, b: 73, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, }"] [2025/04/06 08:24:48.755 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 74, b: 74, c: o, ahRhZ: VgauzvAfkJ, d: 08:25:48, }"] [row2="{ a: 74, b: 74, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:25:48, }"] wait process cdc.test exit for 2-th time... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 06 Apr 2025 00:24:48 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/9d7077c4-d482-42fd-b85e-bd9b18f0d9ec {UpstreamID:7489989421666590001 Namespace:default ID:9d7077c4-d482-42fd-b85e-bd9b18f0d9ec SinkURI:mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1 CreateTime:2025-04-06 08:24:31.34006381 +0800 CST StartTs:457152678123012101 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc004513cb0 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457152678149226502} {CheckpointTs:457152680141520919 MinTableBarrierTs:457152680141520919 AdminJobType:noop} span: {tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}, resolvedTs: 457152680141520919, checkpointTs: 457152680141520919, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c0198952-dccd-454c-b7bd-2f4387081f58 {"id":"c0198952-dccd-454c-b7bd-2f4387081f58","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899085} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087bdb38de c0198952-dccd-454c-b7bd-2f4387081f58 /tidb/cdc/default/default/changefeed/info/9d7077c4-d482-42fd-b85e-bd9b18f0d9ec {"upstream-id":7489989421666590001,"namespace":"default","changefeed-id":"9d7077c4-d482-42fd-b85e-bd9b18f0d9ec","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2025-04-06T08:24:31.34006381+08:00","start-ts":457152678123012101,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457152678149226502} /tidb/cdc/default/default/changefeed/status/9d7077c4-d482-42fd-b85e-bd9b18f0d9ec {"checkpoint-ts":457152680141520919,"min-table-barrier-ts":457152680141520919,"admin-job-type":0} /tidb/cdc/default/default/task/position/c0198952-dccd-454c-b7bd-2f4387081f58/9d7077c4-d482-42fd-b85e-bd9b18f0d9ec {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489989421666590001 {"id":7489989421666590001,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/9d7077c4-d482-42fd-b85e-bd9b18f0d9ec {UpstreamID:7489989421666590001 Namespace:default ID:9d7077c4-d482-42fd-b85e-bd9b18f0d9ec SinkURI:mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1 CreateTime:2025-04-06 08:24:31.34006381 +0800 CST StartTs:457152678123012101 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc004513cb0 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457152678149226502} {CheckpointTs:457152680141520919 MinTableBarrierTs:457152680141520919 AdminJobType:noop} span: {tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}, resolvedTs: 457152680141520919, checkpointTs: 457152680141520919, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c0198952-dccd-454c-b7bd-2f4387081f58 {"id":"c0198952-dccd-454c-b7bd-2f4387081f58","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899085} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087bdb38de c0198952-dccd-454c-b7bd-2f4387081f58 /tidb/cdc/default/default/changefeed/info/9d7077c4-d482-42fd-b85e-bd9b18f0d9ec {"upstream-id":7489989421666590001,"namespace":"default","changefeed-id":"9d7077c4-d482-42fd-b85e-bd9b18f0d9ec","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2025-04-06T08:24:31.34006381+08:00","start-ts":457152678123012101,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457152678149226502} /tidb/cdc/default/default/changefeed/status/9d7077c4-d482-42fd-b85e-bd9b18f0d9ec {"checkpoint-ts":457152680141520919,"min-table-barrier-ts":457152680141520919,"admin-job-type":0} /tidb/cdc/default/default/task/position/c0198952-dccd-454c-b7bd-2f4387081f58/9d7077c4-d482-42fd-b85e-bd9b18f0d9ec {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489989421666590001 {"id":7489989421666590001,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/9d7077c4-d482-42fd-b85e-bd9b18f0d9ec {UpstreamID:7489989421666590001 Namespace:default ID:9d7077c4-d482-42fd-b85e-bd9b18f0d9ec SinkURI:mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1 CreateTime:2025-04-06 08:24:31.34006381 +0800 CST StartTs:457152678123012101 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc004513cb0 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457152678149226502} {CheckpointTs:457152680141520919 MinTableBarrierTs:457152680141520919 AdminJobType:noop} span: {tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}, resolvedTs: 457152680141520919, checkpointTs: 457152680141520919, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c0198952-dccd-454c-b7bd-2f4387081f58 {"id":"c0198952-dccd-454c-b7bd-2f4387081f58","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899085} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087bdb38de c0198952-dccd-454c-b7bd-2f4387081f58 /tidb/cdc/default/default/changefeed/info/9d7077c4-d482-42fd-b85e-bd9b18f0d9ec {"upstream-id":7489989421666590001,"namespace":"default","changefeed-id":"9d7077c4-d482-42fd-b85e-bd9b18f0d9ec","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2025-04-06T08:24:31.34006381+08:00","start-ts":457152678123012101,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457152678149226502} /tidb/cdc/default/default/changefeed/status/9d7077c4-d482-42fd-b85e-bd9b18f0d9ec {"checkpoint-ts":457152680141520919,"min-table-barrier-ts":457152680141520919,"admin-job-type":0} /tidb/cdc/default/default/task/position/c0198952-dccd-454c-b7bd-2f4387081f58/9d7077c4-d482-42fd-b85e-bd9b18f0d9ec {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489989421666590001 + grep -q 'etcd info' {"id":7489989421666590001,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x check_capture_count http://127.0.0.1:2379 1 run task successfully [2025/04/06 08:24:48.855 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ ahRhZ: VgauzvAfkJ, d: 08:25:50, a: 76, b: 76, c: o, }"] [row2="{ a: 76, b: 76, c: D, ahRhZ: Ib7wiR02X4TY4y, d: 08:25:50, }"] [2025/04/06 08:24:48.955 +08:00] [WARN] [diff.go:937] ["find different row"] [column=ahRhZ] [row1="{ a: 77, b: 77, c: o, ahRhZ: VgauzvAfkJ, d: 08:25:51, }"] [row2="{ b: 77, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:25:51, a: 77, }"] table test.t2 not exists for 8-th check, retry later [2025/04/06 08:24:49.055 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 79, b: 79, c: o, ahRhZ: VgauzvAfkJ, d: 08:25:53, }"] [row2="{ a: 79, b: 79, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:25:53, }"] [2025/04/06 08:24:49.156 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ ahRhZ: VgauzvAfkJ, d: 08:25:54, a: 80, b: 80, c: o, }"] [row2="{ a: 80, b: 80, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:25:54, }"] [2025/04/06 08:24:49.256 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 81, b: 81, c: o, ahRhZ: VgauzvAfkJ, d: 08:25:55, }"] [row2="{ b: 81, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:25:55, a: 81, }"] table consistent_replicate_storage_file.check1 not exists for 12-th check, retry later [Sun Apr 6 08:24:49 CST 2025] <<<<<< START cdc server in cli_with_auth case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_with_auth.1167011672.out server --log-file /tmp/tidb_cdc_test/cli_with_auth/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/cli_with_auth/cdc_data --cluster-id default --config /tmp/tidb_cdc_test/cli_with_auth/server.toml + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Sun Apr 6 08:24:49 CST 2025] <<<<<< run test case sink_hang success! >>>>>> Debugger for raftstore-v2 is used Debugger for raftstore-v2 is used [2025/04/06 08:24:49.356 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 82, b: 82, c: o, ahRhZ: VgauzvAfkJ, d: 08:25:56, }"] [row2="{ a: 82, b: 82, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:25:56, }"] [2025/04/06 08:24:49.456 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 84, b: 84, c: o, ahRhZ: VgauzvAfkJ, d: 08:25:58, }"] [row2="{ b: 84, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:25:58, a: 84, }"] changefeed is working normally rts: 457152682327015400->457152682589159400 checkpoint: 457152682327015400->457152682589159400 run task successfully check_ddl_executed /tmp/tidb_cdc_test/ddl_reentrant/cdc.log /tmp/tidb_cdc_test/ddl_reentrant/ddl_temp.sql true [2025/04/06 08:24:43.372 +08:00] [INFO] [mysql_ddl_sink.go:272] ["Exec DDL succeeded"] [namespace=default] [changefeed=a2d89c91-f91a-41f7-8ca2-14e9437fcb8d] [duration=31.176156ms] [sql="CREATE TABLE `ddl_reentrant`.`t3` (`a` INT PRIMARY KEY,`b` INT) PARTITION BY RANGE (`a`) (PARTITION `p0` VALUES LESS THAN (1000),PARTITION `p1` VALUES LESS THAN (2000))"] run task successfully nonempty select id, val from test.availability1 where id=2 and val=2 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later [2025/04/06 08:24:49.556 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 85, b: 85, c: o, ahRhZ: VgauzvAfkJ, d: 08:25:59, }"] [row2="{ ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:25:59, a: 85, b: 85, c: D, }"] [2025/04/06 08:24:49.656 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ d: 08:26:00, a: 86, b: 86, c: o, ahRhZ: VgauzvAfkJ, }"] [row2="{ a: 86, b: 86, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:26:00, }"] [2025/04/06 08:24:49.757 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ d: 08:27:59, a: 87, b: 87, c: o, ahRhZ: VgauzvAfkJ, }"] [row2="{ a: 87, b: 87, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:27:59, }"] [2025/04/06 08:24:49.857 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ d: 08:26:02, a: 88, b: 88, c: o, ahRhZ: VgauzvAfkJ, }"] [row2="{ a: 88, b: 88, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:26:02, }"] [2025/04/06 08:24:49.957 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ d: 08:26:03, a: 89, b: 89, c: o, ahRhZ: VgauzvAfkJ, }"] [row2="{ d: 08:26:03, a: 89, b: 89, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, }"] resume changefeed a2d89c91-f91a-41f7-8ca2-14e9437fcb8d from 457152681252225048 check_ts_forward a2d89c91-f91a-41f7-8ca2-14e9437fcb8d [2025/04/06 08:24:50.057 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ ahRhZ: VgauzvAfkJ, d: 08:26:04, a: 90, b: 90, c: o, }"] [row2="{ d: 08:26:04, a: 90, b: 90, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, }"] [2025/04/06 08:24:50.157 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 91, b: 91, c: o, ahRhZ: VgauzvAfkJ, d: 08:26:05, }"] [row2="{ c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:26:05, a: 91, b: 91, }"] [2025/04/06 08:24:50.258 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ c: o, ahRhZ: VgauzvAfkJ, d: 08:26:06, a: 92, b: 92, }"] [row2="{ a: 92, b: 92, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:26:06, }"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2025/04/06 08:24:50.358 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ d: 08:26:07, a: 93, b: 93, c: o, ahRhZ: VgauzvAfkJ, }"] [row2="{ a: 93, b: 93, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:26:07, }"] [2025/04/06 08:24:50.458 +08:00] [WARN] [diff.go:937] ["find different row"] [column=ahRhZ] [row1="{ d: 08:26:08, a: 94, b: 94, c: o, ahRhZ: VgauzvAfkJ, }"] [row2="{ ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:26:08, a: 94, b: 94, c: D, }"] [2025/04/06 08:24:50.558 +08:00] [WARN] [diff.go:937] ["find different row"] [column=ahRhZ] [row1="{ ahRhZ: VgauzvAfkJ, d: 08:26:09, a: 95, b: 95, c: o, }"] [row2="{ d: 08:26:09, a: 95, b: 95, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, }"] [2025/04/06 08:24:50.658 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ b: 96, c: o, ahRhZ: VgauzvAfkJ, d: 08:26:10, a: 96, }"] [row2="{ a: 96, b: 96, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:26:10, }"] [2025/04/06 08:24:50.758 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ b: 97, c: o, ahRhZ: VgauzvAfkJ, d: 08:26:11, a: 97, }"] [row2="{ a: 97, b: 97, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:26:11, }"] start tidb cluster in /tmp/tidb_cdc_test/sql_mode Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/ddl_attributes Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 [2025/04/06 08:24:50.859 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 98, b: 98, c: o, ahRhZ: VgauzvAfkJ, d: 08:26:12, }"] [row2="{ ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:26:12, a: 98, b: 98, c: D, }"] [2025/04/06 08:24:50.959 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ b: 99, c: o, ahRhZ: VgauzvAfkJ, d: 08:26:13, a: 99, }"] [row2="{ c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:26:13, a: 99, b: 99, }"] [2025/04/06 08:24:51.059 +08:00] [WARN] [diff.go:937] ["find different row"] [column=ahRhZ] [row1="{ a: 101, b: 101, c: o, ahRhZ: VgauzvAfkJ, d: 08:26:15, }"] [row2="{ d: 08:26:15, a: 101, b: 101, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, }"] table consistent_replicate_storage_file.check1 not exists for 13-th check, retry later Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... [2025/04/06 08:24:51.159 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 102, b: 102, c: o, ahRhZ: VgauzvAfkJ, d: 08:26:16, }"] [row2="{ b: 102, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:26:16, a: 102, }"] [2025/04/06 08:24:51.259 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 103, b: 103, c: o, ahRhZ: VgauzvAfkJ, d: 08:26:17, }"] [row2="{ ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:26:17, a: 103, b: 103, c: D, }"] Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release [2025/04/06 08:24:51.360 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ ahRhZ: VgauzvAfkJ, d: 08:26:18, a: 104, b: 104, c: o, }"] [row2="{ d: 08:26:18, a: 104, b: 104, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, }"] [2025/04/06 08:24:51.460 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ d: 08:26:20, a: 106, b: 106, c: o, ahRhZ: VgauzvAfkJ, }"] [row2="{ a: 106, b: 106, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:26:20, }"] [2025/04/06 08:24:51.560 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ b: 107, c: o, ahRhZ: VgauzvAfkJ, d: 08:26:21, a: 107, }"] [row2="{ c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:26:21, a: 107, b: 107, }"] table test.t2 not exists for 9-th check, retry later nonempty select id, val from test.availability1 where id=2 and val=2 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 2-th time, retry later [2025/04/06 08:24:51.660 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 108, b: 108, c: o, ahRhZ: VgauzvAfkJ, d: 08:26:22, }"] [row2="{ c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:26:22, a: 108, b: 108, }"] [2025/04/06 08:24:51.760 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 109, b: 109, c: o, ahRhZ: VgauzvAfkJ, d: 08:26:23, }"] [row2="{ d: 08:26:23, a: 109, b: 109, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, }"] run task failed 1-th time, retry later [2025/04/06 08:24:51.860 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 110, b: 110, c: o, ahRhZ: VgauzvAfkJ, d: 08:26:24, }"] [row2="{ b: 110, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:26:24, a: 110, }"] [2025/04/06 08:24:51.961 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 111, b: 111, c: o, ahRhZ: VgauzvAfkJ, d: 08:32:51, }"] [row2="{ a: 111, b: 111, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:32:51, }"] [2025/04/06 08:24:52.061 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 112, b: 112, c: o, ahRhZ: VgauzvAfkJ, d: 08:26:27, }"] [row2="{ c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:26:27, a: 112, b: 112, }"] [2025/04/06 08:24:52.161 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ ahRhZ: VgauzvAfkJ, d: 08:26:26, a: 113, b: 113, c: o, }"] [row2="{ b: 113, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:26:26, a: 113, }"] [2025/04/06 08:24:52.261 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ c: o, ahRhZ: VgauzvAfkJ, d: 08:26:29, a: 115, b: 115, }"] [row2="{ a: 115, b: 115, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:26:29, }"] [2025/04/06 08:24:52.361 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ b: 116, c: o, ahRhZ: VgauzvAfkJ, d: 08:26:30, a: 116, }"] [row2="{ a: 116, b: 116, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:26:30, }"] [2025/04/06 08:24:52.462 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 117, b: 117, c: o, ahRhZ: VgauzvAfkJ, d: 08:26:32, }"] [row2="{ c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:26:32, a: 117, b: 117, }"] [2025/04/06 08:24:52.562 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 118, b: 118, c: o, ahRhZ: VgauzvAfkJ, d: 08:26:31, }"] [row2="{ ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:26:31, a: 118, b: 118, c: D, }"] Verifying downstream PD is started... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 06 Apr 2025 00:24:52 GMT < Content-Length: 864 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/28ffb967-0ec0-4f61-bed1-3242321101f2 {"id":"28ffb967-0ec0-4f61-bed1-3242321101f2","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899089} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087c1f7eea 28ffb967-0ec0-4f61-bed1-3242321101f2 /tidb/cdc/default/default/upstream/7489989501405550986 {"id":7489989501405550986,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/28ffb967-0ec0-4f61-bed1-3242321101f2 {"id":"28ffb967-0ec0-4f61-bed1-3242321101f2","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899089} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087c1f7eea 28ffb967-0ec0-4f61-bed1-3242321101f2 /tidb/cdc/default/default/upstream/7489989501405550986 {"id":7489989501405550986,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/28ffb967-0ec0-4f61-bed1-3242321101f2 {"id":"28ffb967-0ec0-4f61-bed1-3242321101f2","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899089} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087c1f7eea 28ffb967-0ec0-4f61-bed1-3242321101f2 /tidb/cdc/default/default/upstream/7489989501405550986 {"id":7489989501405550986,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_with_auth.cli.11722.out cli changefeed create --start-ts=457152682220584961 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ --tz=Asia/Shanghai -c=custom-changefeed-name [2025/04/06 08:24:52.662 +08:00] [WARN] [diff.go:937] ["find different row"] [column=ahRhZ] [row1="{ b: 119, c: o, ahRhZ: VgauzvAfkJ, d: 08:26:33, a: 119, }"] [row2="{ a: 119, b: 119, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:26:33, }"] [2025/04/06 08:24:52.762 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ c: o, ahRhZ: VgauzvAfkJ, d: 08:26:34, a: 120, b: 120, }"] [row2="{ a: 120, b: 120, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:26:34, }"] [WARN] --tz is deprecated in changefeed settings. Create changefeed successfully! ID: custom-changefeed-name Info: {"upstream_id":7489989501405550986,"namespace":"default","id":"custom-changefeed-name","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-06T08:24:52.86033655+08:00","start_ts":457152682220584961,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457152682220584961,"checkpoint_ts":457152682220584961,"checkpoint_time":"2025-04-06 08:24:46.840"} PASS ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table consistent_replicate_storage_file.check1 not exists for 14-th check, retry later [2025/04/06 08:24:52.862 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 121, b: 121, c: o, ahRhZ: VgauzvAfkJ, d: 08:26:35, }"] [row2="{ a: 121, b: 121, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:26:35, }"] [2025/04/06 08:24:52.963 +08:00] [WARN] [diff.go:937] ["find different row"] [column=ahRhZ] [row1="{ a: 123, b: 122, c: o, ahRhZ: VgauzvAfkJ, d: 08:26:36, }"] [row2="{ c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:26:36, a: 123, b: 122, }"] [2025/04/06 08:24:53.063 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ ahRhZ: VgauzvAfkJ, d: 08:26:39, a: 125, b: 125, c: o, }"] [row2="{ ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:26:39, a: 125, b: 125, c: D, }"] + run_case_with_unavailable_tikv conf/changefeed-redo.toml + rm -rf /tmp/tidb_cdc_test/synced_status_with_redo + mkdir -p /tmp/tidb_cdc_test/synced_status_with_redo + start_tidb_cluster --workdir /tmp/tidb_cdc_test/synced_status_with_redo shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory The 1 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory coverage: 2.4% of statements in github.com/pingcap/tiflow/... [2025/04/06 08:24:53.163 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ d: 08:26:40, a: 126, b: 126, c: o, ahRhZ: VgauzvAfkJ, }"] [row2="{ a: 126, b: 126, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:26:40, }"] [2025/04/06 08:24:53.263 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ d: 08:26:43, a: 129, b: 129, c: o, ahRhZ: VgauzvAfkJ, }"] [row2="{ a: 129, b: 129, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:26:43, }"] Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.t2 not exists for 10-th check, retry later [2025/04/06 08:24:53.363 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 130, b: 130, c: o, ahRhZ: VgauzvAfkJ, d: 08:29:22, }"] [row2="{ a: 130, b: 130, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:29:22, }"] [2025/04/06 08:24:53.464 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ ahRhZ: VgauzvAfkJ, d: 08:26:45, a: 131, b: 131, c: o, }"] [row2="{ ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:26:45, a: 131, b: 131, c: D, }"] [2025/04/06 08:24:53.564 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 132, b: 132, c: o, ahRhZ: VgauzvAfkJ, d: 08:26:46, }"] [row2="{ d: 08:26:46, a: 132, b: 132, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, }"] [2025/04/06 08:24:53.664 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ d: 08:26:47, a: 133, b: 133, c: o, ahRhZ: VgauzvAfkJ, }"] [row2="{ a: 133, b: 133, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:26:47, }"] [2025/04/06 08:24:53.764 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ d: 08:26:48, a: 134, b: 134, c: o, ahRhZ: VgauzvAfkJ, }"] [row2="{ a: 134, b: 134, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:26:48, }"] pass check, checkpoint tso not forward after 10s run task successfully wait process 8197 exit for 1-th time... wait process 8197 exit for 2-th time... wait process 8197 exit for 3-th time... wait process 8197 exit for 4-th time... /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (8197) - No such process wait process 8197 exit for 5-th time... process 8197 already exit [Sun Apr 6 08:24:47 CST 2025] <<<<<< START cdc server in ddl_only_block_related_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_only_block_related_table.88088810.out server --log-file /tmp/tidb_cdc_test/ddl_only_block_related_table/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_only_block_related_table/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 06 Apr 2025 00:24:50 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/ddl-only-block-related-table {UpstreamID:7489989193297080848 Namespace:default ID:ddl-only-block-related-table SinkURI:mysql://root@127.0.0.1:3306/ CreateTime:2025-04-06 08:23:37.983731343 +0800 CST StartTs:457152664145494017 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc001978e10 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457152664158601224} {CheckpointTs:457152668025225244 MinTableBarrierTs:457152682823254022 AdminJobType:noop} span: {tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}, resolvedTs: 457152668025225244, checkpointTs: 457152668025225244, state: Preparing span: {tableID:116, startKey:7480000000000000ff745f720000000000fa, endKey:7480000000000000ff745f730000000000fa}, resolvedTs: 457152668025225244, checkpointTs: 457152668025225244, state: Preparing span: {tableID:118, startKey:7480000000000000ff765f720000000000fa, endKey:7480000000000000ff765f730000000000fa}, resolvedTs: 457152668025225244, checkpointTs: 457152668025225244, state: Preparing span: {tableID:120, startKey:7480000000000000ff785f720000000000fa, endKey:7480000000000000ff785f730000000000fa}, resolvedTs: 457152668025225244, checkpointTs: 457152668025225244, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8652f271-1c2b-4de4-8015-56bfd678bfa4 {"id":"8652f271-1c2b-4de4-8015-56bfd678bfa4","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899087} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087b0812a2 8652f271-1c2b-4de4-8015-56bfd678bfa4 /tidb/cdc/default/default/changefeed/info/ddl-only-block-related-table {"upstream-id":7489989193297080848,"namespace":"default","changefeed-id":"ddl-only-block-related-table","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2025-04-06T08:23:37.983731343+08:00","start-ts":457152664145494017,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457152664158601224} /tidb/cdc/default/default/changefeed/status/ddl-only-block-related-table {"checkpoint-ts":457152668025225244,"min-table-barrier-ts":457152682823254022,"admin-job-type":0} /tidb/cdc/default/default/task/position/8652f271-1c2b-4de4-8015-56bfd678bfa4/ddl-only-block-related-table {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489989193297080848 {"id":7489989193297080848,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-only-block-related-table {UpstreamID:7489989193297080848 Namespace:default ID:ddl-only-block-related-table SinkURI:mysql://root@127.0.0.1:3306/ CreateTime:2025-04-06 08:23:37.983731343 +0800 CST StartTs:457152664145494017 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc001978e10 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457152664158601224} {CheckpointTs:457152668025225244 MinTableBarrierTs:457152682823254022 AdminJobType:noop} span: {tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}, resolvedTs: 457152668025225244, checkpointTs: 457152668025225244, state: Preparing span: {tableID:116, startKey:7480000000000000ff745f720000000000fa, endKey:7480000000000000ff745f730000000000fa}, resolvedTs: 457152668025225244, checkpointTs: 457152668025225244, state: Preparing span: {tableID:118, startKey:7480000000000000ff765f720000000000fa, endKey:7480000000000000ff765f730000000000fa}, resolvedTs: 457152668025225244, checkpointTs: 457152668025225244, state: Preparing span: {tableID:120, startKey:7480000000000000ff785f720000000000fa, endKey:7480000000000000ff785f730000000000fa}, resolvedTs: 457152668025225244, checkpointTs: 457152668025225244, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8652f271-1c2b-4de4-8015-56bfd678bfa4 {"id":"8652f271-1c2b-4de4-8015-56bfd678bfa4","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899087} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087b0812a2 8652f271-1c2b-4de4-8015-56bfd678bfa4 /tidb/cdc/default/default/changefeed/info/ddl-only-block-related-table {"upstream-id":7489989193297080848,"namespace":"default","changefeed-id":"ddl-only-block-related-table","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2025-04-06T08:23:37.983731343+08:00","start-ts":457152664145494017,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457152664158601224} /tidb/cdc/default/default/changefeed/status/ddl-only-block-related-table {"checkpoint-ts":457152668025225244,"min-table-barrier-ts":457152682823254022,"admin-job-type":0} /tidb/cdc/default/default/task/position/8652f271-1c2b-4de4-8015-56bfd678bfa4/ddl-only-block-related-table {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489989193297080848 {"id":7489989193297080848,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-only-block-related-table {UpstreamID:7489989193297080848 Namespace:default ID:ddl-only-block-related-table SinkURI:mysql://root@127.0.0.1:3306/ CreateTime:2025-04-06 08:23:37.983731343 +0800 CST StartTs:457152664145494017 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc001978e10 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457152664158601224} {CheckpointTs:457152668025225244 MinTableBarrierTs:457152682823254022 AdminJobType:noop} span: {tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}, resolvedTs: 457152668025225244, checkpointTs: 457152668025225244, state: Preparing span: {tableID:116, startKey:7480000000000000ff745f720000000000fa, endKey:7480000000000000ff745f730000000000fa}, resolvedTs: 457152668025225244, checkpointTs: 457152668025225244, state: Preparing span: {tableID:118, startKey:7480000000000000ff765f720000000000fa, endKey:7480000000000000ff765f730000000000fa}, resolvedTs: 457152668025225244, checkpointTs: 457152668025225244, state: Preparing span: {tableID:120, startKey:7480000000000000ff785f720000000000fa, endKey:7480000000000000ff785f730000000000fa}, resolvedTs: 457152668025225244, checkpointTs: 457152668025225244, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8652f271-1c2b-4de4-8015-56bfd678bfa4 {"id":"8652f271-1c2b-4de4-8015-56bfd678bfa4","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899087} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087b0812a2 8652f271-1c2b-4de4-8015-56bfd678bfa4 /tidb/cdc/default/default/changefeed/info/ddl-only-block-related-table {"upstream-id":7489989193297080848,"namespace":"default","changefeed-id":"ddl-only-block-related-table","sink-uri":"mysql://root@127.0.0.1:3306/","create-time":"2025-04-06T08:23:37.983731343+08:00","start-ts":457152664145494017,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457152664158601224} /tidb/cdc/default/default/changefeed/status/ddl-only-block-related-table {"checkpoint-ts":457152668025225244,"min-table-barrier-ts":457152682823254022,"admin-job-type":0} /tidb/cdc/default/default/task/position/8652f271-1c2b-4de4-8015-56bfd678bfa4/ddl-only-block-related-table {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489989193297080848 + grep -q 'etcd info' {"id":7489989193297080848,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x check diff successfully check_ts_forward ddl-only-block-related-table changefeed is working normally rts: 457152668025225200->457152683360649200 checkpoint: 457152668025225200->457152683360649200 run task successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... [2025/04/06 08:24:53.864 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 135, b: 135, c: o, ahRhZ: VgauzvAfkJ, d: 08:26:49, }"] [row2="{ ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:26:49, a: 135, b: 135, c: D, }"] [2025/04/06 08:24:53.965 +08:00] [WARN] [diff.go:937] ["find different row"] [column=ahRhZ] [row1="{ a: 138, b: 138, c: o, ahRhZ: VgauzvAfkJ, d: 08:26:52, }"] [row2="{ b: 138, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:26:52, a: 138, }"] [2025/04/06 08:24:54.065 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ b: 139, c: o, ahRhZ: VgauzvAfkJ, d: 08:33:13, a: 139, }"] [row2="{ a: 139, b: 139, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:33:13, }"] [2025/04/06 08:24:54.165 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 140, b: 140, c: o, ahRhZ: VgauzvAfkJ, d: 08:26:54, }"] [row2="{ ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:26:54, a: 140, b: 140, c: D, }"] [2025/04/06 08:24:54.265 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ c: o, ahRhZ: VgauzvAfkJ, d: 08:26:55, a: 141, b: 141, }"] [row2="{ b: 141, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:26:55, a: 141, }"] check_ts_forward a2d89c91-f91a-41f7-8ca2-14e9437fcb8d + set +x table test.simple not exists for 1-th check, retry later [2025/04/06 08:24:54.366 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 142, b: 142, c: o, ahRhZ: VgauzvAfkJ, d: 08:26:56, }"] [row2="{ a: 142, b: 142, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:26:56, }"] [2025/04/06 08:24:54.466 +08:00] [WARN] [diff.go:937] ["find different row"] [column=ahRhZ] [row1="{ ahRhZ: VgauzvAfkJ, d: 08:26:57, a: 143, b: 143, c: o, }"] [row2="{ c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:26:57, a: 143, b: 143, }"] [2025/04/06 08:24:54.566 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 144, b: 144, c: o, ahRhZ: VgauzvAfkJ, d: 08:26:58, }"] [row2="{ a: 144, b: 144, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:26:58, }"] cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Sun Apr 6 08:24:54 CST 2025] <<<<<< run test case ddl_only_block_related_table success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2025/04/06 08:24:54.666 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ b: 145, c: o, ahRhZ: VgauzvAfkJ, d: 08:26:59, a: 145, }"] [row2="{ a: 145, b: 145, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:26:59, }"] [2025/04/06 08:24:54.766 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 147, b: 147, c: o, ahRhZ: VgauzvAfkJ, d: 08:27:01, }"] [row2="{ b: 147, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:27:01, a: 147, }"] [2025/04/06 08:24:54.866 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 148, b: 148, c: o, ahRhZ: VgauzvAfkJ, d: 08:27:02, }"] [row2="{ b: 148, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:27:02, a: 148, }"] [2025/04/06 08:24:54.967 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 149, b: 149, c: o, ahRhZ: VgauzvAfkJ, d: 08:27:03, }"] [row2="{ b: 149, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:27:03, a: 149, }"] [2025/04/06 08:24:55.067 +08:00] [WARN] [diff.go:937] ["find different row"] [column=ahRhZ] [row1="{ d: 08:27:05, a: 151, b: 151, c: o, ahRhZ: VgauzvAfkJ, }"] [row2="{ a: 151, b: 151, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:27:05, }"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2025/04/06 08:24:55.167 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ b: 152, c: o, ahRhZ: VgauzvAfkJ, d: 08:27:06, a: 152, }"] [row2="{ a: 152, b: 152, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:27:06, }"] [2025/04/06 08:24:55.267 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ d: 08:27:07, a: 153, b: 153, c: o, ahRhZ: VgauzvAfkJ, }"] [row2="{ a: 153, b: 153, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:27:07, }"] [2025/04/06 08:24:55.367 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ ahRhZ: VgauzvAfkJ, d: 08:27:08, a: 154, b: 154, c: o, }"] [row2="{ b: 154, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:27:08, a: 154, }"] table test.t2 not exists for 11-th check, retry later table consistent_replicate_storage_file.check1 not exists for 15-th check, retry later [2025/04/06 08:24:55.467 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 155, b: 155, c: o, ahRhZ: VgauzvAfkJ, d: 08:27:09, }"] [row2="{ a: 155, b: 155, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:27:09, }"] [2025/04/06 08:24:55.568 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ ahRhZ: VgauzvAfkJ, d: 08:27:10, a: 156, b: 156, c: o, }"] [row2="{ d: 08:27:10, a: 156, b: 156, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, }"] chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/synced_status_with_redo Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... nonempty select id, val from test.availability1 where id=2 and val=2 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 3-th time, retry later changefeed is working normally rts: 457152683900141600->457152684424167400 checkpoint: 457152683900141600->457152684266881000 run task successfully check_ddl_executed /tmp/tidb_cdc_test/ddl_reentrant/cdc.log /tmp/tidb_cdc_test/ddl_reentrant/ddl_temp.sql false Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release [2025/04/06 08:24:55.668 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 157, b: 157, c: o, ahRhZ: VgauzvAfkJ, d: 08:27:11, }"] [row2="{ a: 157, b: 157, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:27:11, }"] [2025/04/06 08:24:55.768 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ ahRhZ: VgauzvAfkJ, d: 08:27:13, a: 159, b: 159, c: o, }"] [row2="{ c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:27:13, a: 159, b: 159, }"] [2025/04/06 08:24:55.868 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ c: o, ahRhZ: VgauzvAfkJ, d: 08:29:54, a: 160, b: 160, }"] [row2="{ a: 160, b: 160, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:29:54, }"] Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... [2025/04/06 08:24:50.352 +08:00] [INFO] [mysql_ddl_sink.go:150] ["Execute DDL failed, but error can be ignored"] [startTs=457152681252225044] [ddl="CREATE TABLE `ddl_reentrant`.`t3` (`a` INT PRIMARY KEY,`b` INT) PARTITION BY RANGE (`a`) (PARTITION `p0` VALUES LESS THAN (1000),PARTITION `p1` VALUES LESS THAN (2000))"] [namespace=default] [changefeed=a2d89c91-f91a-41f7-8ca2-14e9437fcb8d] [error="Error 1050 (42S01): Table 'ddl_reentrant.t3' already exists"] run task successfully ------------------------------------------ test ddl create table ddl_reentrant.t4 (a int primary key, b int), is_reentrant: false restored_sql: CREATE TABLE `ddl_reentrant`.`t4` (`a` INT PRIMARY KEY,`b` INT) [2025/04/06 08:24:55.968 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 161, b: 161, c: o, ahRhZ: VgauzvAfkJ, d: 08:32:39, }"] [row2="{ a: 161, b: 161, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:32:39, }"] [2025/04/06 08:24:56.069 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ d: 08:27:16, a: 162, b: 162, c: o, ahRhZ: VgauzvAfkJ, }"] [row2="{ a: 162, b: 162, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:27:16, }"] [2025/04/06 08:24:56.169 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ d: 08:27:17, a: 163, b: 163, c: o, ahRhZ: VgauzvAfkJ, }"] [row2="{ d: 08:27:17, a: 163, b: 163, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, }"] [2025/04/06 08:24:56.269 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ ahRhZ: VgauzvAfkJ, d: 08:27:18, a: 164, b: 164, c: o, }"] [row2="{ c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:27:18, a: 164, b: 164, }"] [2025/04/06 08:24:56.369 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ ahRhZ: VgauzvAfkJ, d: 08:27:20, a: 166, b: 166, c: o, }"] [row2="{ b: 166, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:27:20, a: 166, }"] table test.simple exists table test.`simple-dash` not exists for 1-th check, retry later [2025/04/06 08:24:56.469 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ d: 08:27:21, a: 167, b: 167, c: o, ahRhZ: VgauzvAfkJ, }"] [row2="{ c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:27:21, a: 167, b: 167, }"] [2025/04/06 08:24:56.570 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ c: o, ahRhZ: VgauzvAfkJ, d: 08:27:22, a: 168, b: 168, }"] [row2="{ a: 168, b: 168, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:27:22, }"] check_ts_forward a2d89c91-f91a-41f7-8ca2-14e9437fcb8d ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2025/04/06 08:24:56.670 +08:00] [WARN] [diff.go:940] ["target had superfluous data"] [row="{ a: 169, b: 169, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:29:01, }"] [2025/04/06 08:24:56.770 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 170, b: 170, c: o, ahRhZ: VgauzvAfkJ, d: 08:27:25, }"] [row2="{ ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:27:25, a: 170, b: 170, c: D, }"] [2025/04/06 08:24:56.870 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ c: o, ahRhZ: VgauzvAfkJ, d: 08:27:26, a: 171, b: 171, }"] [row2="{ a: 171, b: 171, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:27:26, }"] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489989557917883646 TiDB Cluster ID. tikv_gc_leader_uuid 65821f1c3a80008 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-9-92vm1, pid:13015, start at 2025-04-06 08:24:55.288992222 +0800 CST m=+2.368164399 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:26:55.297 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:24:55.330 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:14:55.330 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489989557917883646 TiDB Cluster ID. tikv_gc_leader_uuid 65821f1c3a80008 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-9-92vm1, pid:13015, start at 2025-04-06 08:24:55.288992222 +0800 CST m=+2.368164399 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:26:55.297 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:24:55.330 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:14:55.330 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489989555900617424 TiDB Cluster ID. tikv_gc_leader_uuid 65821f1c4880007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-9-92vm1, pid:13085, start at 2025-04-06 08:24:55.34466417 +0800 CST m=+2.365021127 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:26:55.352 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:24:55.330 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:14:55.330 +0800 All versions after safe point can be accessed. (DO NOT EDIT) [2025/04/06 08:24:56.970 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 173, b: 173, c: o, ahRhZ: VgauzvAfkJ, d: 08:27:28, }"] [row2="{ d: 08:27:28, a: 173, b: 173, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, }"] [2025/04/06 08:24:57.071 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 175, b: 175, c: o, ahRhZ: VgauzvAfkJ, d: 08:33:12, }"] [row2="{ ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:33:12, a: 175, b: 175, c: 4Xalj4, }"] Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... table test.t2 not exists for 12-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/syncpoint_check_ts/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/syncpoint_check_ts/tiflash/log/error.log arg matches is ArgMatches { args: {"log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/syncpoint_check_ts/tiflash/log/proxy.log"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/syncpoint_check_ts/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/syncpoint_check_ts/tiflash/db/proxy"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [2025/04/06 08:24:57.171 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ c: o, ahRhZ: VgauzvAfkJ, d: 08:27:31, a: 176, b: 176, }"] [row2="{ c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:27:31, a: 176, b: 176, }"] [2025/04/06 08:24:57.271 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 177, b: 177, c: o, ahRhZ: VgauzvAfkJ, d: 08:27:32, }"] [row2="{ ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:27:32, a: 177, b: 177, c: D, }"] [2025/04/06 08:24:57.371 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 178, b: 178, c: o, ahRhZ: VgauzvAfkJ, d: 08:27:33, }"] [row2="{ a: 178, b: 178, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:27:33, }"] table consistent_replicate_storage_file.check1 not exists for 16-th check, retry later Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2025/04/06 08:24:57.471 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ d: 08:27:34, a: 179, b: 179, c: o, ahRhZ: VgauzvAfkJ, }"] [row2="{ a: 179, b: 179, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:27:34, }"] [2025/04/06 08:24:57.572 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ b: 180, c: o, ahRhZ: VgauzvAfkJ, d: 08:27:35, a: 180, }"] [row2="{ a: 180, b: 180, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:27:35, }"] [2025/04/06 08:24:57.672 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ c: o, ahRhZ: VgauzvAfkJ, d: 08:27:38, a: 183, b: 183, }"] [row2="{ c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:27:38, a: 183, b: 183, }"] [2025/04/06 08:24:57.772 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 184, b: 184, c: o, ahRhZ: VgauzvAfkJ, d: 08:27:39, }"] [row2="{ c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:27:39, a: 184, b: 184, }"] [2025/04/06 08:24:57.872 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 185, b: 185, c: o, ahRhZ: VgauzvAfkJ, d: 08:27:40, }"] [row2="{ d: 08:27:40, a: 185, b: 185, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, }"] [2025/04/06 08:24:57.972 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ b: 186, c: o, ahRhZ: VgauzvAfkJ, d: 08:27:41, a: 186, }"] [row2="{ ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:27:41, a: 186, b: 186, c: D, }"] [2025/04/06 08:24:58.073 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 187, b: 187, c: o, ahRhZ: VgauzvAfkJ, d: 08:27:42, }"] [row2="{ a: 187, b: 187, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:27:42, }"] [2025/04/06 08:24:58.173 +08:00] [WARN] [diff.go:937] ["find different row"] [column=ahRhZ] [row1="{ a: 188, b: 188, c: o, ahRhZ: VgauzvAfkJ, d: 08:27:43, }"] [row2="{ c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:27:43, a: 188, b: 188, }"] [2025/04/06 08:24:58.273 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 189, b: 189, c: o, ahRhZ: VgauzvAfkJ, d: 08:27:44, }"] [row2="{ a: 189, b: 189, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:27:44, }"] [2025/04/06 08:24:58.373 +08:00] [WARN] [diff.go:937] ["find different row"] [column=ahRhZ] [row1="{ ahRhZ: VgauzvAfkJ, d: 08:27:46, a: 190, b: 190, c: o, }"] [row2="{ a: 190, b: 190, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:27:46, }"] Verifying downstream PD is started... changefeed is working normally rts: 457152684424167400->457152684686311400 checkpoint: 457152684424167400->457152684581453800 run task successfully check_ddl_executed /tmp/tidb_cdc_test/ddl_reentrant/cdc.log /tmp/tidb_cdc_test/ddl_reentrant/ddl_temp.sql true [2025/04/06 08:24:57.918 +08:00] [INFO] [mysql_ddl_sink.go:272] ["Exec DDL succeeded"] [namespace=default] [changefeed=a2d89c91-f91a-41f7-8ca2-14e9437fcb8d] [duration=27.537065ms] [sql="CREATE TABLE `ddl_reentrant`.`t4` (`a` INT PRIMARY KEY,`b` INT)"] run task successfully ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2025/04/06 08:24:58.473 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ ahRhZ: VgauzvAfkJ, d: 08:27:47, a: 191, b: 191, c: o, }"] [row2="{ a: 191, b: 191, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:27:47, }"] [2025/04/06 08:24:58.573 +08:00] [WARN] [diff.go:937] ["find different row"] [column=ahRhZ] [row1="{ c: o, ahRhZ: VgauzvAfkJ, d: 08:27:48, a: 192, b: 192, }"] [row2="{ d: 08:27:48, a: 192, b: 192, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, }"] resume changefeed a2d89c91-f91a-41f7-8ca2-14e9437fcb8d from 457152684581453853 check_ts_forward a2d89c91-f91a-41f7-8ca2-14e9437fcb8d wait process 9070 exit for 1-th time... wait process 9070 exit for 2-th time... wait process 9070 exit for 3-th time... process 9070 already exit check_capture_count http://127.0.0.1:2379 0 parse error: Invalid numeric literal at line 1, column 6 run task successfully [Sun Apr 6 08:24:58 CST 2025] <<<<<< START cdc server in kill_owner_with_ddl case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sink/ddlsink/mysql/MySQLSinkExecDDLDelay=return(true);github.com/pingcap/tiflow/cdc/capture/ownerFlushIntervalInject=return(10)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kill_owner_with_ddl.92259227.out server --log-file /tmp/tidb_cdc_test/kill_owner_with_ddl/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/kill_owner_with_ddl/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table test.`simple-dash` exists + endpoints=http://127.0.0.1:2379 + changefeed_id=custom-changefeed-name + expected_state=normal + error_msg=null + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c custom-changefeed-name -s + info='{ "upstream_id": 7489989501405550986, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 457152682744610821, "checkpoint_time": "2025-04-06 08:24:48.839", "error": null }' + echo '{ "upstream_id": 7489989501405550986, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 457152682744610821, "checkpoint_time": "2025-04-06 08:24:48.839", "error": null }' { "upstream_id": 7489989501405550986, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 457152682744610821, "checkpoint_time": "2025-04-06 08:24:48.839", "error": null } ++ echo '{' '"upstream_id":' 7489989501405550986, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 457152682744610821, '"checkpoint_time":' '"2025-04-06' '08:24:48.839",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7489989501405550986, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 457152682744610821, '"checkpoint_time":' '"2025-04-06' '08:24:48.839",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] changefeed count 1 check pass, pd_addr: http://127.0.0.1:2379 [2025/04/06 08:24:58.674 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 193, b: 193, c: o, ahRhZ: VgauzvAfkJ, d: 08:27:49, }"] [row2="{ a: 193, b: 193, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:27:49, }"] [2025/04/06 08:24:58.774 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 194, b: 194, c: o, ahRhZ: VgauzvAfkJ, d: 08:27:50, }"] [row2="{ c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:27:50, a: 194, b: 194, }"] [2025/04/06 08:24:58.874 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 195, b: 195, c: o, ahRhZ: VgauzvAfkJ, d: 08:27:51, }"] [row2="{ a: 195, b: 195, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:27:51, }"] changefeed count 1 check pass, pd_addr: http://127.0.0.1:2679 [2025/04/06 08:24:58.974 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 196, b: 196, c: o, ahRhZ: VgauzvAfkJ, d: 08:27:52, }"] [row2="{ a: 196, b: 196, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:27:52, }"] [2025/04/06 08:24:59.074 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 198, b: 198, c: o, ahRhZ: VgauzvAfkJ, d: 08:27:54, }"] [row2="{ c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:27:54, a: 198, b: 198, }"] [2025/04/06 08:24:59.175 +08:00] [WARN] [diff.go:937] ["find different row"] [column=ahRhZ] [row1="{ a: 199, b: 199, c: o, ahRhZ: VgauzvAfkJ, d: 08:27:55, }"] [row2="{ d: 08:27:55, a: 199, b: 199, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, }"] Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table consistent_replicate_storage_file.check1 not exists for 17-th check, retry later \033[0;36m<<< Run all test success >>>\033[0m ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) changefeed count 1 check pass, pd_addr: http://127.0.0.1:2779 changefeed count 1 check pass, pd_addr: http://127.0.0.1:2379,http://127.0.0.1:2679,http://127.0.0.1:2779 [Pipeline] } [2025/04/06 08:24:59.275 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 200, b: 200, c: o, ahRhZ: VgauzvAfkJ, d: 08:28:01, }"] [row2="{ a: 200, b: 200, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:28:01, }"] [2025/04/06 08:24:59.375 +08:00] [WARN] [diff.go:937] ["find different row"] [column=ahRhZ] [row1="{ ahRhZ: VgauzvAfkJ, d: 08:28:02, a: 201, b: 201, c: o, }"] [row2="{ b: 201, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:28:02, a: 201, }"] [Pipeline] // timeout [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-periodics_cdc_integration_mysql_test-234/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [2025/04/06 08:24:59.475 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ b: 202, c: o, ahRhZ: VgauzvAfkJ, d: 08:28:03, a: 202, }"] [row2="{ b: 202, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:28:03, a: 202, }"] [2025/04/06 08:24:59.575 +08:00] [WARN] [diff.go:940] ["target had superfluous data"] [row="{ a: 203, b: 203, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:28:04, }"] [2025/04/06 08:24:59.676 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 205, b: 205, c: o, ahRhZ: VgauzvAfkJ, d: 08:31:57, }"] [row2="{ a: 205, b: 205, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:31:57, }"] [Pipeline] // dir table test.t2 exists check diff successfully [Pipeline] } Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release [Pipeline] // stage + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.syncpoint_check_ts.cli.14351.out cli tso query --pd=http://127.0.0.1:2379 [Pipeline] } [2025/04/06 08:24:59.776 +08:00] [WARN] [diff.go:937] ["find different row"] [column=ahRhZ] [row1="{ a: 206, b: 206, c: o, ahRhZ: VgauzvAfkJ, d: 08:28:07, }"] [row2="{ a: 206, b: 206, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:28:07, }"] [2025/04/06 08:24:59.876 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ b: 207, c: o, ahRhZ: VgauzvAfkJ, d: 08:28:08, a: 207, }"] [row2="{ c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:28:08, a: 207, b: 207, }"] wait process cdc.test exit for 1-th time... [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [2025/04/06 08:24:59.976 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 208, b: 208, c: o, ahRhZ: VgauzvAfkJ, d: 08:28:09, }"] [row2="{ c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:28:09, a: 208, b: 208, }"] [2025/04/06 08:25:00.076 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 209, b: 209, c: o, ahRhZ: VgauzvAfkJ, d: 08:28:10, }"] [row2="{ b: 209, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:28:10, a: 209, }"] [2025/04/06 08:25:00.177 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ d: 08:28:11, a: 210, b: 210, c: o, ahRhZ: VgauzvAfkJ, }"] [row2="{ a: 210, b: 210, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:28:11, }"] [Pipeline] // node [Pipeline] } Error: [CDC:ErrChangefeedUpdateRefused]changefeed update error: can only update changefeed config when it is stopped or failed update changefeed config should fail when changefeed is running, got Diff of changefeed config: {Type:update Path:[Config CaseSensitive] From:false To:true} {Type:update Path:[Config Scheduler EnableTableAcrossNodes] From:false To:true} + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_with_auth.cli.12006.out cli changefeed --changefeed-id custom-changefeed-name pause [Pipeline] // podTemplate [Pipeline] } [2025/04/06 08:25:00.277 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ d: 08:28:12, a: 211, b: 211, c: o, ahRhZ: VgauzvAfkJ, }"] [row2="{ b: 211, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:28:12, a: 211, }"] [2025/04/06 08:25:00.377 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ ahRhZ: VgauzvAfkJ, d: 08:28:13, a: 212, b: 212, c: o, }"] [row2="{ b: 212, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:28:13, a: 212, }"] [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage wait process cdc.test exit for 2-th time... PASS [Pipeline] } run task failed 1-th time, retry later [2025/04/06 08:25:00.477 +08:00] [WARN] [diff.go:937] ["find different row"] [column=ahRhZ] [row1="{ ahRhZ: VgauzvAfkJ, d: 08:28:15, a: 214, b: 214, c: o, }"] [row2="{ d: 08:28:15, a: 214, b: 214, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, }"] [2025/04/06 08:25:00.577 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ c: o, ahRhZ: VgauzvAfkJ, d: 08:28:16, a: 215, b: 215, }"] [row2="{ a: 215, b: 215, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:28:16, }"] [2025/04/06 08:25:00.677 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 216, b: 216, c: o, ahRhZ: VgauzvAfkJ, d: 08:28:17, }"] [row2="{ d: 08:28:17, a: 216, b: 216, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, }"] coverage: 1.9% of statements in github.com/pingcap/tiflow/... [2025/04/06 08:25:00.778 +08:00] [WARN] [diff.go:937] ["find different row"] [column=ahRhZ] [row1="{ ahRhZ: VgauzvAfkJ, d: 08:28:19, a: 218, b: 218, c: o, }"] [row2="{ a: 218, b: 218, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:28:19, }"] [2025/04/06 08:25:00.878 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ c: o, ahRhZ: VgauzvAfkJ, d: 08:28:20, a: 219, b: 219, }"] [row2="{ a: 219, b: 219, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:28:20, }"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Sun Apr 6 08:25:00 CST 2025] <<<<<< run test case resolve_lock success! >>>>>> table consistent_replicate_storage_file.check1 exists table consistent_replicate_storage_file.t2 not exists for 1-th check, retry later [2025/04/06 08:25:00.978 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ d: 08:31:06, a: 220, b: 220, c: o, ahRhZ: VgauzvAfkJ, }"] [row2="{ c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:31:06, a: 220, b: 220, }"] [2025/04/06 08:25:01.078 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 221, b: 221, c: o, ahRhZ: VgauzvAfkJ, d: 08:28:22, }"] [row2="{ ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:28:22, a: 221, b: 221, c: D, }"] [2025/04/06 08:25:01.178 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 222, b: 222, c: o, ahRhZ: VgauzvAfkJ, d: 08:28:23, }"] [row2="{ a: 222, b: 222, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:28:23, }"] + set +x + tso='457152685584678913 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 457152685584678913 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Sun Apr 6 08:25:01 CST 2025] <<<<<< START cdc server in syncpoint_check_ts case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.syncpoint_check_ts.1438414386.out server --log-file /tmp/tidb_cdc_test/syncpoint_check_ts/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/syncpoint_check_ts/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 06 Apr 2025 00:25:01 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/9d7077c4-d482-42fd-b85e-bd9b18f0d9ec {UpstreamID:7489989421666590001 Namespace:default ID:9d7077c4-d482-42fd-b85e-bd9b18f0d9ec SinkURI:mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1 CreateTime:2025-04-06 08:24:31.34006381 +0800 CST StartTs:457152678123012101 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc000a95b00 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457152678149226502} {CheckpointTs:457152680141520919 MinTableBarrierTs:457152680141520919 AdminJobType:noop} span: {tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}, resolvedTs: 457152680141520919, checkpointTs: 457152680141520919, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b4bf3168-e17d-4b89-93da-4b9b3a140f5c {"id":"b4bf3168-e17d-4b89-93da-4b9b3a140f5c","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899098} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087bdb396d b4bf3168-e17d-4b89-93da-4b9b3a140f5c /tidb/cdc/default/default/changefeed/info/9d7077c4-d482-42fd-b85e-bd9b18f0d9ec {"upstream-id":7489989421666590001,"namespace":"default","changefeed-id":"9d7077c4-d482-42fd-b85e-bd9b18f0d9ec","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2025-04-06T08:24:31.34006381+08:00","start-ts":457152678123012101,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457152678149226502} /tidb/cdc/default/default/changefeed/status/9d7077c4-d482-42fd-b85e-bd9b18f0d9ec {"checkpoint-ts":457152680141520919,"min-table-barrier-ts":457152680141520919,"admin-job-type":0} /tidb/cdc/default/default/task/position/b4bf3168-e17d-4b89-93da-4b9b3a140f5c/9d7077c4-d482-42fd-b85e-bd9b18f0d9ec {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489989421666590001 {"id":7489989421666590001,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/9d7077c4-d482-42fd-b85e-bd9b18f0d9ec {UpstreamID:7489989421666590001 Namespace:default ID:9d7077c4-d482-42fd-b85e-bd9b18f0d9ec SinkURI:mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1 CreateTime:2025-04-06 08:24:31.34006381 +0800 CST StartTs:457152678123012101 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc000a95b00 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457152678149226502} {CheckpointTs:457152680141520919 MinTableBarrierTs:457152680141520919 AdminJobType:noop} span: {tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}, resolvedTs: 457152680141520919, checkpointTs: 457152680141520919, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b4bf3168-e17d-4b89-93da-4b9b3a140f5c {"id":"b4bf3168-e17d-4b89-93da-4b9b3a140f5c","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899098} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087bdb396d b4bf3168-e17d-4b89-93da-4b9b3a140f5c /tidb/cdc/default/default/changefeed/info/9d7077c4-d482-42fd-b85e-bd9b18f0d9ec {"upstream-id":7489989421666590001,"namespace":"default","changefeed-id":"9d7077c4-d482-42fd-b85e-bd9b18f0d9ec","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2025-04-06T08:24:31.34006381+08:00","start-ts":457152678123012101,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457152678149226502} /tidb/cdc/default/default/changefeed/status/9d7077c4-d482-42fd-b85e-bd9b18f0d9ec {"checkpoint-ts":457152680141520919,"min-table-barrier-ts":457152680141520919,"admin-job-type":0} /tidb/cdc/default/default/task/position/b4bf3168-e17d-4b89-93da-4b9b3a140f5c/9d7077c4-d482-42fd-b85e-bd9b18f0d9ec {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489989421666590001 + grep -q 'failed to get info:' {"id":7489989421666590001,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/9d7077c4-d482-42fd-b85e-bd9b18f0d9ec {UpstreamID:7489989421666590001 Namespace:default ID:9d7077c4-d482-42fd-b85e-bd9b18f0d9ec SinkURI:mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1 CreateTime:2025-04-06 08:24:31.34006381 +0800 CST StartTs:457152678123012101 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc000a95b00 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457152678149226502} {CheckpointTs:457152680141520919 MinTableBarrierTs:457152680141520919 AdminJobType:noop} span: {tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}, resolvedTs: 457152680141520919, checkpointTs: 457152680141520919, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b4bf3168-e17d-4b89-93da-4b9b3a140f5c {"id":"b4bf3168-e17d-4b89-93da-4b9b3a140f5c","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899098} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087bdb396d b4bf3168-e17d-4b89-93da-4b9b3a140f5c /tidb/cdc/default/default/changefeed/info/9d7077c4-d482-42fd-b85e-bd9b18f0d9ec {"upstream-id":7489989421666590001,"namespace":"default","changefeed-id":"9d7077c4-d482-42fd-b85e-bd9b18f0d9ec","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2025-04-06T08:24:31.34006381+08:00","start-ts":457152678123012101,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457152678149226502} /tidb/cdc/default/default/changefeed/status/9d7077c4-d482-42fd-b85e-bd9b18f0d9ec {"checkpoint-ts":457152680141520919,"min-table-barrier-ts":457152680141520919,"admin-job-type":0} /tidb/cdc/default/default/task/position/b4bf3168-e17d-4b89-93da-4b9b3a140f5c/9d7077c4-d482-42fd-b85e-bd9b18f0d9ec {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489989421666590001 {"id":7489989421666590001,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x check_capture_count http://127.0.0.1:2379 1 [2025/04/06 08:25:01.279 +08:00] [WARN] [diff.go:937] ["find different row"] [column=ahRhZ] [row1="{ a: 223, b: 223, c: o, ahRhZ: VgauzvAfkJ, d: 08:28:24, }"] [row2="{ a: 223, b: 223, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:28:24, }"] [2025/04/06 08:25:01.379 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ b: 224, c: o, ahRhZ: VgauzvAfkJ, d: 08:28:25, a: 224, }"] [row2="{ a: 224, b: 224, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:28:25, }"] run task successfully VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489989357300557951 TiDB Cluster ID. tikv_gc_leader_uuid 65821ef08600007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-j-shst4, pid:9157, start at 2025-04-06 08:24:10.546273365 +0800 CST m=+3.817043453 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:26:10.556 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:24:10.520 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:14:10.520 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489989357300557951 TiDB Cluster ID. tikv_gc_leader_uuid 65821ef08600007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-j-shst4, pid:9157, start at 2025-04-06 08:24:10.546273365 +0800 CST m=+3.817043453 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:26:10.556 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:24:10.520 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:14:10.520 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489989362460687017 TiDB Cluster ID. tikv_gc_leader_uuid 65821ef09fc0002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-j-shst4, pid:9225, start at 2025-04-06 08:24:10.627926606 +0800 CST m=+3.832542938 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:26:10.644 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:24:10.623 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:14:10.623 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 2 times to try to start tidb cluster... [2025/04/06 08:25:01.479 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ d: 08:28:26, a: 225, b: 225, c: o, ahRhZ: VgauzvAfkJ, }"] [row2="{ d: 08:28:26, a: 225, b: 225, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, }"] [2025/04/06 08:25:01.579 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 227, b: 227, c: o, ahRhZ: VgauzvAfkJ, d: 08:28:28, }"] [row2="{ a: 227, b: 227, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:28:28, }"] [2025/04/06 08:25:01.679 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ d: 08:28:29, a: 228, b: 228, c: o, ahRhZ: VgauzvAfkJ, }"] [row2="{ a: 228, b: 228, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:28:29, }"] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489989566081008294 TiDB Cluster ID. tikv_gc_leader_uuid 65821f2062c0037 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-4-tjmd8, pid:25007, start at 2025-04-06 08:24:59.549849128 +0800 CST m=+2.342486972 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:26:59.558 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:24:59.562 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:14:59.562 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489989566081008294 TiDB Cluster ID. tikv_gc_leader_uuid 65821f2062c0037 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-4-tjmd8, pid:25007, start at 2025-04-06 08:24:59.549849128 +0800 CST m=+2.342486972 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:26:59.558 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:24:59.562 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:14:59.562 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489989578235589960 TiDB Cluster ID. tikv_gc_leader_uuid 65821f2062c000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-4-tjmd8, pid:25083, start at 2025-04-06 08:24:59.549917511 +0800 CST m=+2.280901169 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:26:59.561 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:24:59.562 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:14:59.562 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/ddl_attributes/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/ddl_attributes/tiflash/log/error.log arg matches is ArgMatches { args: {"config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/ddl_attributes/tiflash-proxy.toml"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/ddl_attributes/tiflash/db/proxy"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/ddl_attributes/tiflash/log/proxy.log"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [2025/04/06 08:25:01.779 +08:00] [WARN] [diff.go:940] ["target had superfluous data"] [row="{ a: 229, b: 229, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:28:30, }"] [2025/04/06 08:25:01.880 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ ahRhZ: VgauzvAfkJ, d: 08:28:32, a: 231, b: 231, c: o, }"] [row2="{ d: 08:28:32, a: 231, b: 231, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, }"] + set +x [2025/04/06 08:25:01.980 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ ahRhZ: VgauzvAfkJ, d: 08:28:33, a: 232, b: 232, c: o, }"] [row2="{ ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:28:33, a: 232, b: 232, c: D, }"] [2025/04/06 08:25:02.080 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 233, b: 233, c: o, ahRhZ: VgauzvAfkJ, d: 08:28:34, }"] [row2="{ a: 233, b: 233, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:28:34, }"] [2025/04/06 08:25:02.180 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ ahRhZ: VgauzvAfkJ, d: 08:28:35, a: 234, b: 234, c: o, }"] [row2="{ a: 234, b: 234, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:28:35, }"] nonempty select id, val from test.availability1 where id=2 and val=2 run task successfully wait process cdc.test exit for 1-th time... Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2025/04/06 08:25:02.280 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 235, b: 235, c: o, ahRhZ: VgauzvAfkJ, d: 08:28:36, }"] [row2="{ c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:28:36, a: 235, b: 235, }"] [2025/04/06 08:25:02.381 +08:00] [WARN] [diff.go:937] ["find different row"] [column=ahRhZ] [row1="{ ahRhZ: VgauzvAfkJ, d: 08:28:37, a: 236, b: 236, c: o, }"] [row2="{ a: 236, b: 236, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:28:37, }"] check_ts_forward a2d89c91-f91a-41f7-8ca2-14e9437fcb8d [2025/04/06 08:25:02.481 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ d: 08:28:38, a: 237, b: 237, c: o, ahRhZ: VgauzvAfkJ, }"] [row2="{ a: 237, b: 237, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:28:38, }"] [2025/04/06 08:25:02.581 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ d: 08:28:39, a: 238, b: 238, c: o, ahRhZ: VgauzvAfkJ, }"] [row2="{ a: 238, b: 238, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:28:39, }"] [2025/04/06 08:25:02.681 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 239, b: 239, c: o, ahRhZ: VgauzvAfkJ, d: 08:28:40, }"] [row2="{ a: 239, b: 239, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:28:40, }"] wait process cdc.test exit for 2-th time... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2025/04/06 08:25:02.781 +08:00] [WARN] [diff.go:937] ["find different row"] [column=ahRhZ] [row1="{ d: 08:28:41, a: 240, b: 240, c: o, ahRhZ: VgauzvAfkJ, }"] [row2="{ a: 240, b: 240, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:28:41, }"] [2025/04/06 08:25:02.881 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ ahRhZ: VgauzvAfkJ, d: 08:28:42, a: 241, b: 241, c: o, }"] [row2="{ ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:28:42, a: 241, b: 241, c: D, }"] [2025/04/06 08:25:02.982 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 244, b: 244, c: o, ahRhZ: VgauzvAfkJ, d: 08:28:45, }"] [row2="{ d: 08:28:45, a: 244, b: 244, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, }"] [2025/04/06 08:25:03.082 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 245, b: 245, c: o, ahRhZ: VgauzvAfkJ, d: 08:28:46, }"] [row2="{ ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:28:46, a: 245, b: 245, c: D, }"] [2025/04/06 08:25:03.182 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 246, b: 246, c: o, ahRhZ: VgauzvAfkJ, d: 08:28:47, }"] [row2="{ a: 246, b: 246, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:28:47, }"] cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit run test case test_hang_up_capture [Sun Apr 6 08:25:03 CST 2025] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.61546156.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_hang_up_capture.server1.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_hang_up_capture.server1 --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2025/04/06 08:25:03.282 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 247, b: 247, c: o, ahRhZ: VgauzvAfkJ, d: 08:28:48, }"] [row2="{ b: 247, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:28:48, a: 247, }"] [2025/04/06 08:25:03.382 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ ahRhZ: VgauzvAfkJ, d: 08:28:49, a: 248, b: 248, c: o, }"] [row2="{ a: 248, b: 248, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:28:49, }"] [2025/04/06 08:25:03.483 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ ahRhZ: VgauzvAfkJ, d: 08:28:50, a: 249, b: 249, c: o, }"] [row2="{ a: 249, b: 249, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:28:50, }"] table consistent_replicate_storage_file.t2 not exists for 2-th check, retry later + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_attributes.cli.26312.out cli tso query --pd=http://127.0.0.1:2379 [2025/04/06 08:25:03.583 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 250, b: 250, c: o, ahRhZ: VgauzvAfkJ, d: 08:28:51, }"] [row2="{ a: 250, b: 250, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:28:51, }"] [2025/04/06 08:25:03.683 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 251, b: 251, c: o, ahRhZ: VgauzvAfkJ, d: 08:28:52, }"] [row2="{ a: 251, b: 251, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:28:52, }"] [2025/04/06 08:25:03.783 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ d: 08:28:53, a: 252, b: 252, c: o, ahRhZ: VgauzvAfkJ, }"] [row2="{ a: 252, b: 252, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:28:53, }"] [2025/04/06 08:25:03.883 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 253, b: 253, c: o, ahRhZ: VgauzvAfkJ, d: 08:28:54, }"] [row2="{ ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:28:54, a: 253, b: 253, c: D, }"] [2025/04/06 08:25:03.984 +08:00] [WARN] [diff.go:937] ["find different row"] [column=ahRhZ] [row1="{ a: 254, b: 254, c: o, ahRhZ: VgauzvAfkJ, d: 08:28:55, }"] [row2="{ a: 254, b: 254, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:28:55, }"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2025/04/06 08:25:04.084 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 255, b: 255, c: o, ahRhZ: VgauzvAfkJ, d: 08:28:56, }"] [row2="{ a: 255, b: 255, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:28:56, }"] [2025/04/06 08:25:04.184 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 257, b: 257, c: o, ahRhZ: VgauzvAfkJ, d: 08:28:58, }"] [row2="{ a: 257, b: 257, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:28:58, }"] changefeed is working normally rts: 457152684581453800->457152686521581600 checkpoint: 457152684581453800->457152686521581600 run task successfully check_ddl_executed /tmp/tidb_cdc_test/ddl_reentrant/cdc.log /tmp/tidb_cdc_test/ddl_reentrant/ddl_temp.sql false [2025/04/06 08:25:00.792 +08:00] [INFO] [mysql_ddl_sink.go:150] ["Execute DDL failed, but error can be ignored"] [startTs=457152684581453849] [ddl="CREATE TABLE `ddl_reentrant`.`t4` (`a` INT PRIMARY KEY,`b` INT)"] [namespace=default] [changefeed=a2d89c91-f91a-41f7-8ca2-14e9437fcb8d] [error="Error 1050 (42S01): Table 'ddl_reentrant.t4' already exists"] run task successfully ------------------------------------------ test ddl alter table ddl_reentrant.t3 exchange partition p0 with table ddl_reentrant.t4 without validation, is_reentrant: true restored_sql: ALTER TABLE `ddl_reentrant`.`t3` EXCHANGE PARTITION `p0` WITH TABLE `ddl_reentrant`.`t4` WITHOUT VALIDATION + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 06 Apr 2025 00:25:04 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/97a8becc-beb3-4e2c-8030-0600c23c50ae {"id":"97a8becc-beb3-4e2c-8030-0600c23c50ae","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899101} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087c51d5e5 97a8becc-beb3-4e2c-8030-0600c23c50ae /tidb/cdc/default/default/upstream/7489989557917883646 {"id":7489989557917883646,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/97a8becc-beb3-4e2c-8030-0600c23c50ae {"id":"97a8becc-beb3-4e2c-8030-0600c23c50ae","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899101} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087c51d5e5 97a8becc-beb3-4e2c-8030-0600c23c50ae /tidb/cdc/default/default/upstream/7489989557917883646 {"id":7489989557917883646,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/97a8becc-beb3-4e2c-8030-0600c23c50ae {"id":"97a8becc-beb3-4e2c-8030-0600c23c50ae","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899101} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087c51d5e5 97a8becc-beb3-4e2c-8030-0600c23c50ae /tidb/cdc/default/default/upstream/7489989557917883646 {"id":7489989557917883646,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.syncpoint_check_ts.cli.14444.out cli changefeed create --start-ts=457152685584678913 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --config=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/syncpoint_check_ts/conf/changefeed.toml [2025/04/06 08:25:04.284 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ c: o, ahRhZ: VgauzvAfkJ, d: 08:28:59, a: 258, b: 258, }"] [row2="{ a: 258, b: 258, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:28:59, }"] [2025/04/06 08:25:04.384 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ d: 08:29:00, a: 259, b: 259, c: o, ahRhZ: VgauzvAfkJ, }"] [row2="{ a: 259, b: 259, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:29:00, }"] [2025/04/06 08:25:04.485 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ c: o, ahRhZ: VgauzvAfkJ, d: 08:29:02, a: 260, b: 260, }"] [row2="{ d: 08:29:02, a: 260, b: 260, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, }"] =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/changefeed_auto_stop/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... Create changefeed successfully! ID: 81d19c27-d7c3-419b-8b2f-4e16ee2693e1 Info: {"upstream_id":7489989557917883646,"namespace":"default","id":"81d19c27-d7c3-419b-8b2f-4e16ee2693e1","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2025-04-06T08:25:04.616497096+08:00","start_ts":457152685584678913,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":true,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":30000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457152685584678913,"checkpoint_ts":457152685584678913,"checkpoint_time":"2025-04-06 08:24:59.673"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... [2025/04/06 08:25:04.585 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ c: o, ahRhZ: VgauzvAfkJ, d: 08:29:03, a: 261, b: 261, }"] [row2="{ b: 261, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:29:03, a: 261, }"] [2025/04/06 08:25:04.685 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 262, b: 262, c: o, ahRhZ: VgauzvAfkJ, d: 08:29:04, }"] [row2="{ d: 08:29:04, a: 262, b: 262, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, }"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2025/04/06 08:25:04.785 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ d: 08:29:06, a: 264, b: 264, c: o, ahRhZ: VgauzvAfkJ, }"] [row2="{ a: 264, b: 264, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:29:06, }"] [2025/04/06 08:25:04.885 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 265, b: 265, c: o, ahRhZ: VgauzvAfkJ, d: 08:29:07, }"] [row2="{ a: 265, b: 265, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:29:07, }"] [2025/04/06 08:25:04.985 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ d: 08:29:08, a: 266, b: 266, c: o, ahRhZ: VgauzvAfkJ, }"] [row2="{ a: 266, b: 266, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:29:08, }"] [2025/04/06 08:25:05.086 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ b: 267, c: o, ahRhZ: VgauzvAfkJ, d: 08:29:09, a: 267, }"] [row2="{ d: 08:29:09, a: 267, b: 267, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, }"] [2025/04/06 08:25:05.186 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ b: 268, c: o, ahRhZ: VgauzvAfkJ, d: 08:29:10, a: 268, }"] [row2="{ a: 268, b: 268, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:29:10, }"] + endpoints=http://127.0.0.1:2379 + changefeed_id=custom-changefeed-name + expected_state=stopped + error_msg=null + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c custom-changefeed-name -s + info='{ "upstream_id": 7489989501405550986, "namespace": "default", "id": "custom-changefeed-name", "state": "stopped", "checkpoint_tso": 457152685706838025, "checkpoint_time": "2025-04-06 08:25:00.139", "error": null }' + echo '{ "upstream_id": 7489989501405550986, "namespace": "default", "id": "custom-changefeed-name", "state": "stopped", "checkpoint_tso": 457152685706838025, "checkpoint_time": "2025-04-06 08:25:00.139", "error": null }' { "upstream_id": 7489989501405550986, "namespace": "default", "id": "custom-changefeed-name", "state": "stopped", "checkpoint_tso": 457152685706838025, "checkpoint_time": "2025-04-06 08:25:00.139", "error": null } ++ echo '{' '"upstream_id":' 7489989501405550986, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"stopped",' '"checkpoint_tso":' 457152685706838025, '"checkpoint_time":' '"2025-04-06' '08:25:00.139",' '"error":' null '}' ++ jq -r .state + state=stopped + [[ ! stopped == \s\t\o\p\p\e\d ]] ++ echo '{' '"upstream_id":' 7489989501405550986, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"stopped",' '"checkpoint_tso":' 457152685706838025, '"checkpoint_time":' '"2025-04-06' '08:25:00.139",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_with_auth.cli.12085.out cli changefeed update --pd=http://127.0.0.1:2379,http://127.0.0.1:2679,http://127.0.0.1:2779 --config=/tmp/tidb_cdc_test/cli_with_auth/changefeed.toml --no-confirm --changefeed-id custom-changefeed-name [2025/04/06 08:25:05.286 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ ahRhZ: VgauzvAfkJ, d: 08:29:11, a: 269, b: 269, c: o, }"] [row2="{ a: 269, b: 269, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:29:11, }"] [2025/04/06 08:25:05.386 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 270, b: 270, c: o, ahRhZ: VgauzvAfkJ, d: 08:29:12, }"] [row2="{ a: 270, b: 270, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:29:12, }"] [2025/04/06 08:25:05.486 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 271, b: 271, c: o, ahRhZ: VgauzvAfkJ, d: 08:29:13, }"] [row2="{ a: 271, b: 271, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:29:13, }"] table consistent_replicate_storage_file.t2 not exists for 3-th check, retry later + set +x + tso='457152686695907329 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 457152686695907329 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Sun Apr 6 08:25:05 CST 2025] <<<<<< START cdc server in ddl_attributes case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_attributes.2634926351.out server --log-file /tmp/tidb_cdc_test/ddl_attributes/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_attributes/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2025/04/06 08:25:05.587 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ ahRhZ: VgauzvAfkJ, d: 08:29:14, a: 272, b: 272, c: o, }"] [row2="{ a: 272, b: 272, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:29:14, }"] Diff of changefeed config: {Type:update Path:[Config CaseSensitive] From:false To:true} {Type:update Path:[Config Scheduler EnableTableAcrossNodes] From:false To:true} Update changefeed config successfully! ID: custom-changefeed-name Info: {"upstream_id":7489989501405550986,"namespace":"default","id":"custom-changefeed-name","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-06T08:24:52.86033655+08:00","start_ts":457152682220584961,"admin_job_type":1,"config":{"memory_quota":1073741824,"case_sensitive":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"delete_only_output_handle_key_columns":null,"content_compatible":null,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"stopped","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":0,"checkpoint_ts":457152685706838025,"checkpoint_time":"2025-04-06 08:25:00.139"} PASS coverage: 2.9% of statements in github.com/pingcap/tiflow/... + set +x VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489989590850596047 TiDB Cluster ID. tikv_gc_leader_uuid 65821f250f8000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-d-7dnbd, pid:12650, start at 2025-04-06 08:25:04.367863375 +0800 CST m=+2.509303533 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:27:04.377 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:25:04.368 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:15:04.368 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489989590850596047 TiDB Cluster ID. tikv_gc_leader_uuid 65821f250f8000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-d-7dnbd, pid:12650, start at 2025-04-06 08:25:04.367863375 +0800 CST m=+2.509303533 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:27:04.377 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:25:04.368 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:15:04.368 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489989593898171999 TiDB Cluster ID. tikv_gc_leader_uuid 65821f258bc000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-d-7dnbd, pid:12725, start at 2025-04-06 08:25:04.838307869 +0800 CST m=+2.923348854 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:27:04.861 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:25:04.865 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:15:04.865 +0800 All versions after safe point can be accessed. (DO NOT EDIT) [2025/04/06 08:25:05.687 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 273, b: 273, c: o, ahRhZ: VgauzvAfkJ, d: 08:29:15, }"] [row2="{ a: 273, b: 273, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:29:15, }"] [2025/04/06 08:25:05.787 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ c: o, ahRhZ: VgauzvAfkJ, d: 08:29:16, a: 274, b: 274, }"] [row2="{ ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:29:16, a: 274, b: 274, c: D, }"] [2025/04/06 08:25:05.887 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 275, b: 275, c: o, ahRhZ: VgauzvAfkJ, d: 08:29:17, }"] [row2="{ c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:29:17, a: 275, b: 275, }"] [2025/04/06 08:25:05.987 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 276, b: 276, c: o, ahRhZ: VgauzvAfkJ, d: 08:29:18, }"] [row2="{ d: 08:29:18, a: 276, b: 276, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, }"] [2025/04/06 08:25:06.088 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 277, b: 277, c: o, ahRhZ: VgauzvAfkJ, d: 08:29:19, }"] [row2="{ ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:29:19, a: 277, b: 277, c: D, }"] [2025/04/06 08:25:06.188 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 278, b: 278, c: o, ahRhZ: VgauzvAfkJ, d: 08:29:20, }"] [row2="{ c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:29:20, a: 278, b: 278, }"] [2025/04/06 08:25:06.288 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ d: 08:29:21, a: 279, b: 279, c: o, ahRhZ: VgauzvAfkJ, }"] [row2="{ a: 279, b: 279, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:29:21, }"] Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/synced_status_with_redo/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/synced_status_with_redo/tiflash/log/error.log arg matches is ArgMatches { args: {"log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/synced_status_with_redo/tiflash/log/proxy.log"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/synced_status_with_redo/tiflash/db/proxy"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/synced_status_with_redo/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [2025/04/06 08:25:06.388 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ d: 08:29:24, a: 280, b: 280, c: o, ahRhZ: VgauzvAfkJ, }"] [row2="{ c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:29:24, a: 280, b: 280, }"] [2025/04/06 08:25:06.488 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 281, b: 281, c: o, ahRhZ: VgauzvAfkJ, d: 08:29:25, }"] [row2="{ b: 281, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:29:25, a: 281, }"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 06 Apr 2025 00:25:06 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/e684662d-022f-4a79-9ef3-0a59dba96193 {UpstreamID:7489988628380945915 Namespace:default ID:e684662d-022f-4a79-9ef3-0a59dba96193 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-06 08:21:28.032207553 +0800 CST StartTs:457152628763131907 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0026f8120 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457152630100066306} {CheckpointTs:457152687116124165 MinTableBarrierTs:457152687116124165 AdminJobType:noop} span: {tableID:112, startKey:7480000000000000ff705f720000000000fa, endKey:7480000000000000ff705f730000000000fa}, resolvedTs: 457152687129231362, checkpointTs: 457152687116124165, state: Replicating span: {tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}, resolvedTs: 457152687116124166, checkpointTs: 457152687116124165, state: Replicating span: {tableID:116, startKey:7480000000000000ff745f720000000000fa, endKey:7480000000000000ff745f730000000000fa}, resolvedTs: 457152687116124165, checkpointTs: 457152687116124165, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/bdda6b6f-a6ca-44ee-9f6e-288eec9b3c22 {"id":"bdda6b6f-a6ca-44ee-9f6e-288eec9b3c22","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899103} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 3 /tidb/cdc/default/__cdc_meta__/owner/2231960879054f4d bdda6b6f-a6ca-44ee-9f6e-288eec9b3c22 /tidb/cdc/default/default/changefeed/info/e684662d-022f-4a79-9ef3-0a59dba96193 {"upstream-id":7489988628380945915,"namespace":"default","changefeed-id":"e684662d-022f-4a79-9ef3-0a59dba96193","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-06T08:21:28.032207553+08:00","start-ts":457152628763131907,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457152630100066306} /tidb/cdc/default/default/changefeed/status/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":457152687116124165,"min-table-barrier-ts":457152687116124165,"admin-job-type":0} /tidb/cdc/default/default/task/position/bdda6b6f-a6ca-44ee-9f6e-288eec9b3c22/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489988628380945915 {"id":7489988628380945915,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/e684662d-022f-4a79-9ef3-0a59dba96193 {UpstreamID:7489988628380945915 Namespace:default ID:e684662d-022f-4a79-9ef3-0a59dba96193 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-06 08:21:28.032207553 +0800 CST StartTs:457152628763131907 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0026f8120 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457152630100066306} {CheckpointTs:457152687116124165 MinTableBarrierTs:457152687116124165 AdminJobType:noop} span: {tableID:112, startKey:7480000000000000ff705f720000000000fa, endKey:7480000000000000ff705f730000000000fa}, resolvedTs: 457152687129231362, checkpointTs: 457152687116124165, state: Replicating span: {tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}, resolvedTs: 457152687116124166, checkpointTs: 457152687116124165, state: Replicating span: {tableID:116, startKey:7480000000000000ff745f720000000000fa, endKey:7480000000000000ff745f730000000000fa}, resolvedTs: 457152687116124165, checkpointTs: 457152687116124165, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/bdda6b6f-a6ca-44ee-9f6e-288eec9b3c22 {"id":"bdda6b6f-a6ca-44ee-9f6e-288eec9b3c22","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899103} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 3 /tidb/cdc/default/__cdc_meta__/owner/2231960879054f4d bdda6b6f-a6ca-44ee-9f6e-288eec9b3c22 /tidb/cdc/default/default/changefeed/info/e684662d-022f-4a79-9ef3-0a59dba96193 {"upstream-id":7489988628380945915,"namespace":"default","changefeed-id":"e684662d-022f-4a79-9ef3-0a59dba96193","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-06T08:21:28.032207553+08:00","start-ts":457152628763131907,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457152630100066306} /tidb/cdc/default/default/changefeed/status/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":457152687116124165,"min-table-barrier-ts":457152687116124165,"admin-job-type":0} /tidb/cdc/default/default/task/position/bdda6b6f-a6ca-44ee-9f6e-288eec9b3c22/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489988628380945915 {"id":7489988628380945915,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/e684662d-022f-4a79-9ef3-0a59dba96193 {UpstreamID:7489988628380945915 Namespace:default ID:e684662d-022f-4a79-9ef3-0a59dba96193 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-06 08:21:28.032207553 +0800 CST StartTs:457152628763131907 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0026f8120 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457152630100066306} {CheckpointTs:457152687116124165 MinTableBarrierTs:457152687116124165 AdminJobType:noop} span: {tableID:112, startKey:7480000000000000ff705f720000000000fa, endKey:7480000000000000ff705f730000000000fa}, resolvedTs: 457152687129231362, checkpointTs: 457152687116124165, state: Replicating span: {tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}, resolvedTs: 457152687116124166, checkpointTs: 457152687116124165, state: Replicating span: {tableID:116, startKey:7480000000000000ff745f720000000000fa, endKey:7480000000000000ff745f730000000000fa}, resolvedTs: 457152687116124165, checkpointTs: 457152687116124165, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/bdda6b6f-a6ca-44ee-9f6e-288eec9b3c22 {"id":"bdda6b6f-a6ca-44ee-9f6e-288eec9b3c22","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899103} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 3 /tidb/cdc/default/__cdc_meta__/owner/2231960879054f4d bdda6b6f-a6ca-44ee-9f6e-288eec9b3c22 /tidb/cdc/default/default/changefeed/info/e684662d-022f-4a79-9ef3-0a59dba96193 {"upstream-id":7489988628380945915,"namespace":"default","changefeed-id":"e684662d-022f-4a79-9ef3-0a59dba96193","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-06T08:21:28.032207553+08:00","start-ts":457152628763131907,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457152630100066306} /tidb/cdc/default/default/changefeed/status/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":457152687116124165,"min-table-barrier-ts":457152687116124165,"admin-job-type":0} /tidb/cdc/default/default/task/position/bdda6b6f-a6ca-44ee-9f6e-288eec9b3c22/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489988628380945915 {"id":7489988628380945915,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x cdc.test cli capture list 2>&1 | grep '"is-owner": true' [2025/04/06 08:25:06.589 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ ahRhZ: VgauzvAfkJ, d: 08:29:26, a: 282, b: 282, c: o, }"] [row2="{ a: 282, b: 282, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:29:26, }"] [2025/04/06 08:25:06.689 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ d: 08:29:27, a: 283, b: 283, c: o, ahRhZ: VgauzvAfkJ, }"] [row2="{ b: 283, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:29:27, a: 283, }"] [2025/04/06 08:25:06.789 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 284, b: 284, c: o, ahRhZ: VgauzvAfkJ, d: 08:29:28, }"] [row2="{ a: 284, b: 284, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:29:28, }"] check_ts_forward a2d89c91-f91a-41f7-8ca2-14e9437fcb8d [2025/04/06 08:25:06.889 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ ahRhZ: VgauzvAfkJ, d: 08:29:29, a: 285, b: 285, c: o, }"] [row2="{ a: 285, b: 285, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:29:29, }"] [2025/04/06 08:25:06.989 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 287, b: 287, c: o, ahRhZ: VgauzvAfkJ, d: 08:29:31, }"] [row2="{ b: 287, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:29:31, a: 287, }"] + set +x [2025/04/06 08:25:07.090 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ d: 08:29:32, a: 288, b: 288, c: o, ahRhZ: VgauzvAfkJ, }"] [row2="{ d: 08:29:32, a: 288, b: 288, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, }"] [2025/04/06 08:25:07.190 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 289, b: 289, c: o, ahRhZ: VgauzvAfkJ, d: 08:29:33, }"] [row2="{ a: 289, b: 289, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:29:33, }"] [2025/04/06 08:25:07.290 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 290, b: 290, c: o, ahRhZ: VgauzvAfkJ, d: 08:29:34, }"] [row2="{ a: 290, b: 290, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:29:34, }"] table consistent_replicate_storage_file.t2 not exists for 4-th check, retry later + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_with_auth.cli.12120.out cli changefeed --changefeed-id custom-changefeed-name resume [2025/04/06 08:25:07.390 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ ahRhZ: VgauzvAfkJ, d: 08:29:35, a: 291, b: 291, c: o, }"] [row2="{ ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:29:35, a: 291, b: 291, c: D, }"] [2025/04/06 08:25:07.490 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 292, b: 292, c: o, ahRhZ: VgauzvAfkJ, d: 08:29:36, }"] [row2="{ b: 292, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:29:36, a: 292, }"] [2025/04/06 08:25:07.591 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ ahRhZ: VgauzvAfkJ, d: 08:29:37, a: 293, b: 293, c: o, }"] [row2="{ a: 293, b: 293, c: D, ahRhZ: 0Jv1cKkNlGAMJM5UqGCDy, d: 08:29:37, }"] start tidb cluster in /tmp/tidb_cdc_test/changefeed_auto_stop Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... [2025/04/06 08:25:07.691 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 294, b: 294, c: o, ahRhZ: VgauzvAfkJ, d: 08:29:38, }"] [row2="{ a: 294, b: 294, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:29:38, }"] [2025/04/06 08:25:07.791 +08:00] [WARN] [diff.go:937] ["find different row"] [column=ahRhZ] [row1="{ a: 295, b: 295, c: o, ahRhZ: VgauzvAfkJ, d: 08:29:39, }"] [row2="{ c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:29:39, a: 295, b: 295, }"] "is-owner": true, run task successfully [2025/04/06 08:25:07.891 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ c: o, ahRhZ: VgauzvAfkJ, d: 08:29:40, a: 296, b: 296, }"] [row2="{ d: 08:29:40, a: 296, b: 296, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, }"] [2025/04/06 08:25:07.991 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 297, b: 297, c: o, ahRhZ: VgauzvAfkJ, d: 08:29:41, }"] [row2="{ b: 297, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:29:41, a: 297, }"] [2025/04/06 08:25:08.092 +08:00] [WARN] [diff.go:937] ["find different row"] [column=ahRhZ] [row1="{ a: 298, b: 298, c: o, ahRhZ: VgauzvAfkJ, d: 08:29:42, }"] [row2="{ a: 298, b: 298, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:29:42, }"] PASS coverage: 2.1% of statements in github.com/pingcap/tiflow/... [2025/04/06 08:25:08.192 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ c: o, ahRhZ: VgauzvAfkJ, d: 08:29:43, a: 299, b: 299, }"] [row2="{ d: 08:29:43, a: 299, b: 299, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, }"] [2025/04/06 08:25:08.292 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ b: 300, c: o, ahRhZ: VgauzvAfkJ, d: 08:29:44, a: 300, }"] [row2="{ b: 300, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:29:44, a: 300, }"] [2025/04/06 08:25:08.392 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ c: o, ahRhZ: VgauzvAfkJ, d: 08:29:45, a: 301, b: 301, }"] [row2="{ a: 301, b: 301, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:29:45, }"] [2025/04/06 08:25:08.492 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ b: 302, c: o, ahRhZ: VgauzvAfkJ, d: 08:29:46, a: 302, }"] [row2="{ a: 302, b: 302, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:29:46, }"] [2025/04/06 08:25:08.593 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ ahRhZ: VgauzvAfkJ, d: 08:29:47, a: 303, b: 303, c: o, }"] [row2="{ c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:29:47, a: 303, b: 303, }"] changefeed is working normally rts: 457152687150465000->457152687570157600 checkpoint: 457152687268429800->457152687570157600 run task successfully check_ddl_executed /tmp/tidb_cdc_test/ddl_reentrant/cdc.log /tmp/tidb_cdc_test/ddl_reentrant/ddl_temp.sql true [2025/04/06 08:25:06.988 +08:00] [INFO] [mysql_ddl_sink.go:272] ["Exec DDL succeeded"] [namespace=default] [changefeed=a2d89c91-f91a-41f7-8ca2-14e9437fcb8d] [duration=41.70397ms] [sql="ALTER TABLE `ddl_reentrant`.`t3` EXCHANGE PARTITION `p0` WITH TABLE `ddl_reentrant`.`t4` WITHOUT VALIDATION"] run task successfully + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 06 Apr 2025 00:25:08 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0aebdd79-f636-4cdf-afbd-c27299eb68b9 {"id":"0aebdd79-f636-4cdf-afbd-c27299eb68b9","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899105} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087c606fee 0aebdd79-f636-4cdf-afbd-c27299eb68b9 /tidb/cdc/default/default/upstream/7489989566081008294 {"id":7489989566081008294,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0aebdd79-f636-4cdf-afbd-c27299eb68b9 {"id":"0aebdd79-f636-4cdf-afbd-c27299eb68b9","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899105} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087c606fee 0aebdd79-f636-4cdf-afbd-c27299eb68b9 /tidb/cdc/default/default/upstream/7489989566081008294 {"id":7489989566081008294,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0aebdd79-f636-4cdf-afbd-c27299eb68b9 {"id":"0aebdd79-f636-4cdf-afbd-c27299eb68b9","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899105} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087c606fee 0aebdd79-f636-4cdf-afbd-c27299eb68b9 /tidb/cdc/default/default/upstream/7489989566081008294 {"id":7489989566081008294,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_attributes.cli.26418.out cli changefeed create --start-ts=457152686695907329 --sink-uri=mysql://root@127.0.0.1:3306/ Create changefeed successfully! ID: c66d4f2d-c155-4761-bcef-bf49863c8061 Info: {"upstream_id":7489989566081008294,"namespace":"default","id":"c66d4f2d-c155-4761-bcef-bf49863c8061","sink_uri":"mysql://root@127.0.0.1:3306/","create_time":"2025-04-06T08:25:08.832292882+08:00","start_ts":457152686695907329,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457152686695907329,"checkpoint_ts":457152686695907329,"checkpoint_time":"2025-04-06 08:25:03.912"} PASS [2025/04/06 08:25:08.693 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 304, b: 304, c: o, ahRhZ: VgauzvAfkJ, d: 08:38:10, }"] [row2="{ a: 304, b: 304, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:38:10, }"] [2025/04/06 08:25:08.793 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ d: 08:29:49, a: 305, b: 305, c: o, ahRhZ: VgauzvAfkJ, }"] [row2="{ c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:29:49, a: 305, b: 305, }"] + cd /tmp/tidb_cdc_test/synced_status_with_redo ++ run_cdc_cli_tso_query 127.0.0.1 2379 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status_with_redo.cli.14043.out cli tso query --pd=http://127.0.0.1:2379 resume changefeed a2d89c91-f91a-41f7-8ca2-14e9437fcb8d from 457152687268429840 check_ts_forward a2d89c91-f91a-41f7-8ca2-14e9437fcb8d coverage: 2.4% of statements in github.com/pingcap/tiflow/... [2025/04/06 08:25:08.893 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 306, b: 306, c: o, ahRhZ: VgauzvAfkJ, d: 08:29:50, }"] [row2="{ a: 306, b: 306, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:29:50, }"] [2025/04/06 08:25:08.993 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ c: o, ahRhZ: VgauzvAfkJ, d: 08:29:51, a: 307, b: 307, }"] [row2="{ b: 307, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:29:51, a: 307, }"] [2025/04/06 08:25:09.093 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ ahRhZ: VgauzvAfkJ, d: 08:29:52, a: 308, b: 308, c: o, }"] [row2="{ b: 308, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:29:52, a: 308, }"] table consistent_replicate_storage_file.t2 not exists for 5-th check, retry later [2025/04/06 08:25:09.194 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 309, b: 309, c: o, ahRhZ: VgauzvAfkJ, d: 08:29:53, }"] [row2="{ ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:29:53, a: 309, b: 309, c: D, }"] [2025/04/06 08:25:09.294 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 310, b: 310, c: o, ahRhZ: VgauzvAfkJ, d: 08:29:55, }"] [row2="{ b: 310, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:29:55, a: 310, }"] owner pid: 6159 owner id bdda6b6f-a6ca-44ee-9f6e-288eec9b3c22 [Sun Apr 6 08:25:09 CST 2025] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.62726274.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_hang_up_capture.server2.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_hang_up_capture.server2 --cluster-id default --addr 127.0.0.1:8301 ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + set +x [2025/04/06 08:25:09.394 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ ahRhZ: VgauzvAfkJ, d: 08:29:56, a: 311, b: 311, c: o, }"] [row2="{ c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:29:56, a: 311, b: 311, }"] [2025/04/06 08:25:09.494 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 312, b: 312, c: o, ahRhZ: VgauzvAfkJ, d: 08:29:57, }"] [row2="{ ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:29:57, a: 312, b: 312, c: D, }"] [2025/04/06 08:25:09.594 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 313, b: 313, c: o, ahRhZ: VgauzvAfkJ, d: 08:29:58, }"] [row2="{ b: 313, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:29:58, a: 313, }"] [2025/04/06 08:25:09.695 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ d: 08:29:59, a: 314, b: 314, c: o, ahRhZ: VgauzvAfkJ, }"] [row2="{ c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:29:59, a: 314, b: 314, }"] [2025/04/06 08:25:09.795 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ d: 08:30:00, a: 315, b: 315, c: o, ahRhZ: VgauzvAfkJ, }"] [row2="{ b: 315, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:30:00, a: 315, }"] [2025/04/06 08:25:09.895 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 316, b: 316, c: o, ahRhZ: VgauzvAfkJ, d: 08:39:25, }"] [row2="{ a: 316, b: 316, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:39:25, }"] [2025/04/06 08:25:09.995 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ c: o, ahRhZ: VgauzvAfkJ, d: 08:30:02, a: 317, b: 317, }"] [row2="{ ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:30:02, a: 317, b: 317, c: D, }"] [2025/04/06 08:25:10.095 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 318, b: 318, c: o, ahRhZ: VgauzvAfkJ, d: 08:30:03, }"] [row2="{ d: 08:30:03, a: 318, b: 318, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, }"] + set +x + tso='457152687929819138 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + awk -F ' ' '{print $1}' + echo 457152687929819138 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + set +x + start_ts=457152687929819138 + run_cdc_server --workdir /tmp/tidb_cdc_test/synced_status_with_redo --binary cdc.test [Sun Apr 6 08:25:10 CST 2025] <<<<<< START cdc server in synced_status_with_redo case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status_with_redo.1407714079.out server --log-file /tmp/tidb_cdc_test/synced_status_with_redo/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/synced_status_with_redo/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2025/04/06 08:25:10.196 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ ahRhZ: VgauzvAfkJ, d: 08:30:04, a: 319, b: 319, c: o, }"] [row2="{ c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:30:04, a: 319, b: 319, }"] [2025/04/06 08:25:10.296 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 320, b: 320, c: o, ahRhZ: VgauzvAfkJ, d: 08:30:05, }"] [row2="{ b: 320, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:30:05, a: 320, }"] + set +x [2025/04/06 08:25:10.396 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 321, b: 321, c: o, ahRhZ: VgauzvAfkJ, d: 08:30:06, }"] [row2="{ c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:30:06, a: 321, b: 321, }"] [2025/04/06 08:25:10.496 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ d: 08:30:07, a: 322, b: 322, c: o, ahRhZ: VgauzvAfkJ, }"] [row2="{ a: 322, b: 322, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:30:07, }"] [2025/04/06 08:25:10.596 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ b: 323, c: o, ahRhZ: VgauzvAfkJ, d: 08:30:08, a: 323, }"] [row2="{ ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:30:08, a: 323, b: 323, c: D, }"] [2025/04/06 08:25:10.697 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 324, b: 324, c: o, ahRhZ: VgauzvAfkJ, d: 08:30:09, }"] [row2="{ a: 324, b: 324, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:30:09, }"] [2025/04/06 08:25:10.797 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ c: o, ahRhZ: VgauzvAfkJ, d: 08:30:10, a: 325, b: 325, }"] [row2="{ a: 325, b: 325, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:30:10, }"] run task failed 1-th time, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/move_table/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [2025/04/06 08:25:10.897 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 326, b: 326, c: o, ahRhZ: VgauzvAfkJ, d: 08:30:11, }"] [row2="{ d: 08:30:11, a: 326, b: 326, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, }"] [2025/04/06 08:25:10.997 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 327, b: 327, c: o, ahRhZ: VgauzvAfkJ, d: 08:30:12, }"] [row2="{ b: 327, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:30:12, a: 327, }"] [2025/04/06 08:25:11.097 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ c: o, ahRhZ: VgauzvAfkJ, d: 08:30:13, a: 328, b: 328, }"] [row2="{ a: 328, b: 328, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:30:13, }"] [2025/04/06 08:25:11.198 +08:00] [WARN] [diff.go:937] ["find different row"] [column=ahRhZ] [row1="{ a: 329, b: 329, c: o, ahRhZ: VgauzvAfkJ, d: 08:30:14, }"] [row2="{ a: 329, b: 329, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:30:14, }"] [2025/04/06 08:25:11.298 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ c: o, ahRhZ: VgauzvAfkJ, d: 08:30:15, a: 330, b: 330, }"] [row2="{ a: 330, b: 330, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:30:15, }"] [2025/04/06 08:25:11.398 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ ahRhZ: VgauzvAfkJ, d: 08:30:16, a: 331, b: 331, c: o, }"] [row2="{ ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:30:16, a: 331, b: 331, c: D, }"] wait process 9230 exit for 1-th time... wait process 9230 exit for 2-th time... wait process 9230 exit for 3-th time... /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (9230) - No such process wait process 9230 exit for 4-th time... process 9230 already exit check_capture_count http://127.0.0.1:2379 0 [2025/04/06 08:25:11.498 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ d: 08:30:17, a: 332, b: 332, c: o, ahRhZ: VgauzvAfkJ, }"] [row2="{ c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:30:17, a: 332, b: 332, }"] [2025/04/06 08:25:11.598 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 333, b: 333, c: o, ahRhZ: VgauzvAfkJ, d: 08:30:18, }"] [row2="{ a: 333, b: 333, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:30:18, }"] parse error: Invalid numeric literal at line 1, column 6 run task successfully [Sun Apr 6 08:25:11 CST 2025] <<<<<< START cdc server in kill_owner_with_ddl case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sink/ddlsink/mysql/MySQLSinkExecDDLDelay=return(true);github.com/pingcap/tiflow/cdc/capture/ownerFlushIntervalInject=return(10)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kill_owner_with_ddl.93729374.out server --log-file /tmp/tidb_cdc_test/kill_owner_with_ddl/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/kill_owner_with_ddl/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table consistent_replicate_storage_file.t2 not exists for 6-th check, retry later [2025/04/06 08:25:11.699 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ d: 08:30:19, a: 334, b: 334, c: o, ahRhZ: VgauzvAfkJ, }"] [row2="{ b: 334, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:30:19, a: 334, }"] [2025/04/06 08:25:11.799 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 335, b: 335, c: o, ahRhZ: VgauzvAfkJ, d: 08:30:20, }"] [row2="{ d: 08:30:20, a: 335, b: 335, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, }"] [2025/04/06 08:25:11.899 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ b: 336, c: o, ahRhZ: VgauzvAfkJ, d: 08:30:21, a: 336, }"] [row2="{ ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:30:21, a: 336, b: 336, c: D, }"] Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release [2025/04/06 08:25:11.999 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 337, b: 337, c: o, ahRhZ: VgauzvAfkJ, d: 08:30:22, }"] [row2="{ b: 337, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:30:22, a: 337, }"] [2025/04/06 08:25:12.099 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 338, b: 338, c: o, ahRhZ: VgauzvAfkJ, d: 08:30:23, }"] [row2="{ d: 08:30:23, a: 338, b: 338, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, }"] [2025/04/06 08:25:12.200 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ ahRhZ: VgauzvAfkJ, d: 08:30:24, a: 339, b: 339, c: o, }"] [row2="{ c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:30:24, a: 339, b: 339, }"] [2025/04/06 08:25:12.300 +08:00] [WARN] [diff.go:937] ["find different row"] [column=ahRhZ] [row1="{ d: 08:30:25, a: 340, b: 340, c: o, ahRhZ: VgauzvAfkJ, }"] [row2="{ a: 340, b: 340, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:30:25, }"] [2025/04/06 08:25:12.400 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 341, b: 341, c: o, ahRhZ: VgauzvAfkJ, d: 08:30:26, }"] [row2="{ ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:30:26, a: 341, b: 341, c: D, }"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 06 Apr 2025 00:25:12 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/e684662d-022f-4a79-9ef3-0a59dba96193 {UpstreamID:7489988628380945915 Namespace:default ID:e684662d-022f-4a79-9ef3-0a59dba96193 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-06 08:21:28.032207553 +0800 CST StartTs:457152628763131907 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0043125a0 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457152630100066306} {CheckpointTs:457152688688988171 MinTableBarrierTs:457152688688988171 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/43d114ff-40b6-4e5d-bfe3-4fb7abc0dc3a {"id":"43d114ff-40b6-4e5d-bfe3-4fb7abc0dc3a","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899109} /tidb/cdc/default/__cdc_meta__/capture/bdda6b6f-a6ca-44ee-9f6e-288eec9b3c22 {"id":"bdda6b6f-a6ca-44ee-9f6e-288eec9b3c22","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899103} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 3 /tidb/cdc/default/__cdc_meta__/owner/2231960879054f4d bdda6b6f-a6ca-44ee-9f6e-288eec9b3c22 /tidb/cdc/default/__cdc_meta__/owner/2231960879054f97 43d114ff-40b6-4e5d-bfe3-4fb7abc0dc3a /tidb/cdc/default/default/changefeed/info/e684662d-022f-4a79-9ef3-0a59dba96193 {"upstream-id":7489988628380945915,"namespace":"default","changefeed-id":"e684662d-022f-4a79-9ef3-0a59dba96193","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-06T08:21:28.032207553+08:00","start-ts":457152628763131907,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457152630100066306} /tidb/cdc/default/default/changefeed/status/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":457152688688988171,"min-table-barrier-ts":457152688688988171,"admin-job-type":0} /tidb/cdc/default/default/task/position/43d114ff-40b6-4e5d-bfe3-4fb7abc0dc3a/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/bdda6b6f-a6ca-44ee-9f6e-288eec9b3c22/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489988628380945915 {"id":7489988628380945915,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/e684662d-022f-4a79-9ef3-0a59dba96193 {UpstreamID:7489988628380945915 Namespace:default ID:e684662d-022f-4a79-9ef3-0a59dba96193 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-06 08:21:28.032207553 +0800 CST StartTs:457152628763131907 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0043125a0 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457152630100066306} {CheckpointTs:457152688688988171 MinTableBarrierTs:457152688688988171 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/43d114ff-40b6-4e5d-bfe3-4fb7abc0dc3a {"id":"43d114ff-40b6-4e5d-bfe3-4fb7abc0dc3a","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899109} /tidb/cdc/default/__cdc_meta__/capture/bdda6b6f-a6ca-44ee-9f6e-288eec9b3c22 {"id":"bdda6b6f-a6ca-44ee-9f6e-288eec9b3c22","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899103} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 3 /tidb/cdc/default/__cdc_meta__/owner/2231960879054f4d bdda6b6f-a6ca-44ee-9f6e-288eec9b3c22 /tidb/cdc/default/__cdc_meta__/owner/2231960879054f97 43d114ff-40b6-4e5d-bfe3-4fb7abc0dc3a /tidb/cdc/default/default/changefeed/info/e684662d-022f-4a79-9ef3-0a59dba96193 {"upstream-id":7489988628380945915,"namespace":"default","changefeed-id":"e684662d-022f-4a79-9ef3-0a59dba96193","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-06T08:21:28.032207553+08:00","start-ts":457152628763131907,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457152630100066306} /tidb/cdc/default/default/changefeed/status/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":457152688688988171,"min-table-barrier-ts":457152688688988171,"admin-job-type":0} /tidb/cdc/default/default/task/position/43d114ff-40b6-4e5d-bfe3-4fb7abc0dc3a/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/bdda6b6f-a6ca-44ee-9f6e-288eec9b3c22/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489988628380945915 {"id":7489988628380945915,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: changefeedID: default/e684662d-022f-4a79-9ef3-0a59dba96193 {UpstreamID:7489988628380945915 Namespace:default ID:e684662d-022f-4a79-9ef3-0a59dba96193 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-06 08:21:28.032207553 +0800 CST StartTs:457152628763131907 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0043125a0 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457152630100066306} {CheckpointTs:457152688688988171 MinTableBarrierTs:457152688688988171 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/43d114ff-40b6-4e5d-bfe3-4fb7abc0dc3a {"id":"43d114ff-40b6-4e5d-bfe3-4fb7abc0dc3a","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899109} /tidb/cdc/default/__cdc_meta__/capture/bdda6b6f-a6ca-44ee-9f6e-288eec9b3c22 {"id":"bdda6b6f-a6ca-44ee-9f6e-288eec9b3c22","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899103} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 3 /tidb/cdc/default/__cdc_meta__/owner/2231960879054f4d bdda6b6f-a6ca-44ee-9f6e-288eec9b3c22 /tidb/cdc/default/__cdc_meta__/owner/2231960879054f97 43d114ff-40b6-4e5d-bfe3-4fb7abc0dc3a /tidb/cdc/default/default/changefeed/info/e684662d-022f-4a79-9ef3-0a59dba96193 {"upstream-id":7489988628380945915,"namespace":"default","changefeed-id":"e684662d-022f-4a79-9ef3-0a59dba96193","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-06T08:21:28.032207553+08:00","start-ts":457152628763131907,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457152630100066306} /tidb/cdc/default/default/changefeed/status/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":457152688688988171,"min-table-barrier-ts":457152688688988171,"admin-job-type":0} /tidb/cdc/default/default/task/position/43d114ff-40b6-4e5d-bfe3-4fb7abc0dc3a/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/bdda6b6f-a6ca-44ee-9f6e-288eec9b3c22/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489988628380945915 {"id":7489988628380945915,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x cdc.test cli capture list 2>&1 | grep -v "bdda6b6f-a6ca-44ee-9f6e-288eec9b3c22" | grep id [2025/04/06 08:25:12.500 +08:00] [WARN] [diff.go:937] ["find different row"] [column=ahRhZ] [row1="{ d: 08:30:27, a: 342, b: 342, c: o, ahRhZ: VgauzvAfkJ, }"] [row2="{ a: 342, b: 342, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:30:27, }"] [2025/04/06 08:25:12.600 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 343, b: 343, c: o, ahRhZ: VgauzvAfkJ, d: 08:30:28, }"] [row2="{ d: 08:30:28, a: 343, b: 343, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, }"] + endpoints=http://127.0.0.1:2379 + changefeed_id=custom-changefeed-name + expected_state=normal + error_msg=null + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c custom-changefeed-name -s + info='{ "upstream_id": 7489989501405550986, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 457152688590684170, "checkpoint_time": "2025-04-06 08:25:11.140", "error": null }' + echo '{ "upstream_id": 7489989501405550986, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 457152688590684170, "checkpoint_time": "2025-04-06 08:25:11.140", "error": null }' { "upstream_id": 7489989501405550986, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 457152688590684170, "checkpoint_time": "2025-04-06 08:25:11.140", "error": null } ++ echo '{' '"upstream_id":' 7489989501405550986, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 457152688590684170, '"checkpoint_time":' '"2025-04-06' '08:25:11.140",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7489989501405550986, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 457152688590684170, '"checkpoint_time":' '"2025-04-06' '08:25:11.140",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_with_auth.cli.12184.out cli changefeed --changefeed-id custom-changefeed-name remove [2025/04/06 08:25:12.701 +08:00] [WARN] [diff.go:937] ["find different row"] [column=ahRhZ] [row1="{ a: 344, b: 344, c: o, ahRhZ: VgauzvAfkJ, d: 08:30:29, }"] [row2="{ a: 344, b: 344, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:30:29, }"] [2025/04/06 08:25:12.801 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 345, b: 345, c: o, ahRhZ: VgauzvAfkJ, d: 08:30:30, }"] [row2="{ a: 345, b: 345, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:30:30, }"] [2025/04/06 08:25:12.901 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 346, b: 346, c: o, ahRhZ: VgauzvAfkJ, d: 08:30:31, }"] [row2="{ d: 08:30:31, a: 346, b: 346, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, }"] [2025/04/06 08:25:13.001 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ d: 08:30:32, a: 347, b: 347, c: o, ahRhZ: VgauzvAfkJ, }"] [row2="{ d: 08:30:32, a: 347, b: 347, c: rWEy, ahRhZ: QdArKAP2RgOEVYU7Y, }"] [2025/04/06 08:25:13.101 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ c: o, ahRhZ: VgauzvAfkJ, d: 08:30:33, a: 348, b: 348, }"] [row2="{ ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:30:33, a: 348, b: 348, c: D, }"] [2025/04/06 08:25:13.201 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ ahRhZ: VgauzvAfkJ, d: 08:30:34, a: 349, b: 349, c: o, }"] [row2="{ d: 08:30:34, a: 349, b: 349, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, }"] [2025/04/06 08:25:13.302 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ ahRhZ: VgauzvAfkJ, d: 08:30:35, a: 350, b: 350, c: o, }"] [row2="{ a: 350, b: 350, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:30:35, }"] [2025/04/06 08:25:13.402 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 351, b: 351, c: o, ahRhZ: VgauzvAfkJ, d: 08:30:36, }"] [row2="{ d: 08:30:36, a: 351, b: 351, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, }"] check_ts_forward a2d89c91-f91a-41f7-8ca2-14e9437fcb8d + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 06 Apr 2025 00:25:13 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/36001d52-813c-4fb2-b770-e3850bfe2115 {"id":"36001d52-813c-4fb2-b770-e3850bfe2115","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899110} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087c729fe9 36001d52-813c-4fb2-b770-e3850bfe2115 /tidb/cdc/default/default/upstream/7489989590850596047 {"id":7489989590850596047,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/36001d52-813c-4fb2-b770-e3850bfe2115 {"id":"36001d52-813c-4fb2-b770-e3850bfe2115","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899110} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087c729fe9 36001d52-813c-4fb2-b770-e3850bfe2115 /tidb/cdc/default/default/upstream/7489989590850596047 {"id":7489989590850596047,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/36001d52-813c-4fb2-b770-e3850bfe2115 {"id":"36001d52-813c-4fb2-b770-e3850bfe2115","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899110} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087c729fe9 36001d52-813c-4fb2-b770-e3850bfe2115 /tidb/cdc/default/default/upstream/7489989590850596047 {"id":7489989590850596047,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + config_path=conf/changefeed-redo.toml + SINK_URI='mysql://root@127.0.0.1:3306/?max-txn-row=1' + run_cdc_cli changefeed create --start-ts=457152687929819138 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/synced_status_with_redo/conf/changefeed-redo.toml + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status_with_redo.cli.14142.out cli changefeed create --start-ts=457152687929819138 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/synced_status_with_redo/conf/changefeed-redo.toml Changefeed remove successfully. ID: custom-changefeed-name CheckpointTs: 457152688852828170 SinkURI: mysql://normal:xxxxx@127.0.0.1:3306/ PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... [2025/04/06 08:25:13.502 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 352, b: 352, c: o, ahRhZ: VgauzvAfkJ, d: 08:30:37, }"] [row2="{ d: 08:30:37, a: 352, b: 352, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, }"] [2025/04/06 08:25:13.602 +08:00] [WARN] [diff.go:937] ["find different row"] [column=ahRhZ] [row1="{ d: 08:32:38, a: 353, b: 353, c: o, ahRhZ: VgauzvAfkJ, }"] [row2="{ b: 353, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:32:38, a: 353, }"] table consistent_replicate_storage_file.t2 exists Create changefeed successfully! ID: test-1 Info: {"upstream_id":7489989590850596047,"namespace":"default","id":"test-1","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2025-04-06T08:25:13.656368315+08:00","start_ts":457152687929819138,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"eventual","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"storage":"file:///tmp/tidb_cdc_test/synced_status/redo","use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":120,"checkpoint_interval":20}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457152687929819138,"checkpoint_ts":457152687929819138,"checkpoint_time":"2025-04-06 08:25:08.619"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... Starting Upstream TiDB... start tidb cluster in /tmp/tidb_cdc_test/sql_mode Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... [2025/04/06 08:25:13.702 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 354, b: 354, c: o, ahRhZ: VgauzvAfkJ, d: 08:30:39, }"] [row2="{ b: 354, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:30:39, a: 354, }"] [2025/04/06 08:25:13.803 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 355, b: 355, c: o, ahRhZ: VgauzvAfkJ, d: 08:30:40, }"] [row2="{ b: 355, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:30:40, a: 355, }"] [2025/04/06 08:25:13.903 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ b: 356, c: o, ahRhZ: VgauzvAfkJ, d: 08:30:41, a: 356, }"] [row2="{ a: 356, b: 356, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:30:41, }"] "cluster-id": "default" "id": "43d114ff-40b6-4e5d-bfe3-4fb7abc0dc3a", "cluster-id": "default" run task successfully Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2025/04/06 08:25:14.003 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 357, b: 357, c: o, ahRhZ: VgauzvAfkJ, d: 08:30:42, }"] [row2="{ a: 357, b: 357, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:30:42, }"] [2025/04/06 08:25:14.103 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ d: 08:30:43, a: 358, b: 358, c: o, ahRhZ: VgauzvAfkJ, }"] [row2="{ c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:30:43, a: 358, b: 358, }"] check diff successfully start tidb cluster in /tmp/tidb_cdc_test/move_table Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... [2025/04/06 08:25:14.203 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 359, b: 359, c: o, ahRhZ: VgauzvAfkJ, d: 08:30:44, }"] [row2="{ a: 359, b: 359, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:30:44, }"] [2025/04/06 08:25:14.303 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ ahRhZ: VgauzvAfkJ, d: 08:30:46, a: 360, b: 360, c: o, }"] [row2="{ a: 360, b: 360, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:30:46, }"] [2025/04/06 08:25:14.404 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 361, b: 361, c: o, ahRhZ: VgauzvAfkJ, d: 08:30:47, }"] [row2="{ a: 361, b: 361, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:30:47, }"] wait process cdc.test exit for 1-th time... [2025/04/06 08:25:14.504 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ ahRhZ: VgauzvAfkJ, d: 08:30:48, a: 362, b: 362, c: o, }"] [row2="{ ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:30:48, a: 362, b: 362, c: D, }"] [2025/04/06 08:25:14.604 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 363, b: 363, c: o, ahRhZ: VgauzvAfkJ, d: 08:30:49, }"] [row2="{ ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:30:49, a: 363, b: 363, c: D, }"] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489989538505028507 TiDB Cluster ID. tikv_gc_leader_uuid 65821f1a094000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-x-hbn5c, pid:9999, start at 2025-04-06 08:24:53.043582834 +0800 CST m=+4.713594900 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:26:53.055 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:24:53.029 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:14:53.029 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489989538505028507 TiDB Cluster ID. tikv_gc_leader_uuid 65821f1a094000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-x-hbn5c, pid:9999, start at 2025-04-06 08:24:53.043582834 +0800 CST m=+4.713594900 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:26:53.055 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:24:53.029 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:14:53.029 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489989539303037750 TiDB Cluster ID. tikv_gc_leader_uuid 65821f1a778000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-x-hbn5c, pid:10070, start at 2025-04-06 08:24:53.498246139 +0800 CST m=+5.113990350 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:26:53.505 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:24:53.470 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:14:53.470 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/multi_rocks/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/multi_rocks/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/multi_rocks/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/multi_rocks/tiflash/db/proxy"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/multi_rocks/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } changefeed is working normally rts: 457152688880877600->457152689143021600 checkpoint: 457152688880877600->457152689247617000 run task successfully check_ddl_executed /tmp/tidb_cdc_test/ddl_reentrant/cdc.log /tmp/tidb_cdc_test/ddl_reentrant/ddl_temp.sql true [2025/04/06 08:25:10.844 +08:00] [INFO] [mysql_ddl_sink.go:272] ["Exec DDL succeeded"] [namespace=default] [changefeed=a2d89c91-f91a-41f7-8ca2-14e9437fcb8d] [duration=43.254787ms] [sql="ALTER TABLE `ddl_reentrant`.`t3` EXCHANGE PARTITION `p0` WITH TABLE `ddl_reentrant`.`t4` WITHOUT VALIDATION"] run task successfully [2025/04/06 08:25:14.704 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ c: o, ahRhZ: VgauzvAfkJ, d: 08:30:51, a: 365, b: 365, }"] [row2="{ c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:30:51, a: 365, b: 365, }"] [2025/04/06 08:25:14.804 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 366, b: 366, c: o, ahRhZ: VgauzvAfkJ, d: 08:30:52, }"] [row2="{ a: 366, b: 366, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:30:52, }"] [2025/04/06 08:25:14.905 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ c: o, ahRhZ: VgauzvAfkJ, d: 08:30:53, a: 367, b: 367, }"] [row2="{ b: 367, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:30:53, a: 367, }"] + set +x + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 06 Apr 2025 00:25:14 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/9d7077c4-d482-42fd-b85e-bd9b18f0d9ec {UpstreamID:7489989421666590001 Namespace:default ID:9d7077c4-d482-42fd-b85e-bd9b18f0d9ec SinkURI:mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1 CreateTime:2025-04-06 08:24:31.34006381 +0800 CST StartTs:457152678123012101 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003863e60 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457152678149226502} {CheckpointTs:457152680141520919 MinTableBarrierTs:457152680141520919 AdminJobType:noop} span: {tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}, resolvedTs: 457152680141520919, checkpointTs: 457152680141520919, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/292c6474-4b4b-4125-a38d-7f6efa3cd97a {"id":"292c6474-4b4b-4125-a38d-7f6efa3cd97a","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899112} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087bdb39f5 292c6474-4b4b-4125-a38d-7f6efa3cd97a /tidb/cdc/default/default/changefeed/info/9d7077c4-d482-42fd-b85e-bd9b18f0d9ec {"upstream-id":7489989421666590001,"namespace":"default","changefeed-id":"9d7077c4-d482-42fd-b85e-bd9b18f0d9ec","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2025-04-06T08:24:31.34006381+08:00","start-ts":457152678123012101,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457152678149226502} /tidb/cdc/default/default/changefeed/status/9d7077c4-d482-42fd-b85e-bd9b18f0d9ec {"checkpoint-ts":457152680141520919,"min-table-barrier-ts":457152680141520919,"admin-job-type":0} /tidb/cdc/default/default/task/position/292c6474-4b4b-4125-a38d-7f6efa3cd97a/9d7077c4-d482-42fd-b85e-bd9b18f0d9ec {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489989421666590001 {"id":7489989421666590001,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/9d7077c4-d482-42fd-b85e-bd9b18f0d9ec {UpstreamID:7489989421666590001 Namespace:default ID:9d7077c4-d482-42fd-b85e-bd9b18f0d9ec SinkURI:mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1 CreateTime:2025-04-06 08:24:31.34006381 +0800 CST StartTs:457152678123012101 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003863e60 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457152678149226502} {CheckpointTs:457152680141520919 MinTableBarrierTs:457152680141520919 AdminJobType:noop} span: {tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}, resolvedTs: 457152680141520919, checkpointTs: 457152680141520919, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/292c6474-4b4b-4125-a38d-7f6efa3cd97a {"id":"292c6474-4b4b-4125-a38d-7f6efa3cd97a","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899112} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087bdb39f5 292c6474-4b4b-4125-a38d-7f6efa3cd97a /tidb/cdc/default/default/changefeed/info/9d7077c4-d482-42fd-b85e-bd9b18f0d9ec {"upstream-id":7489989421666590001,"namespace":"default","changefeed-id":"9d7077c4-d482-42fd-b85e-bd9b18f0d9ec","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2025-04-06T08:24:31.34006381+08:00","start-ts":457152678123012101,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457152678149226502} /tidb/cdc/default/default/changefeed/status/9d7077c4-d482-42fd-b85e-bd9b18f0d9ec {"checkpoint-ts":457152680141520919,"min-table-barrier-ts":457152680141520919,"admin-job-type":0} /tidb/cdc/default/default/task/position/292c6474-4b4b-4125-a38d-7f6efa3cd97a/9d7077c4-d482-42fd-b85e-bd9b18f0d9ec {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489989421666590001 {"id":7489989421666590001,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/9d7077c4-d482-42fd-b85e-bd9b18f0d9ec {UpstreamID:7489989421666590001 Namespace:default ID:9d7077c4-d482-42fd-b85e-bd9b18f0d9ec SinkURI:mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1 CreateTime:2025-04-06 08:24:31.34006381 +0800 CST StartTs:457152678123012101 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003863e60 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457152678149226502} {CheckpointTs:457152680141520919 MinTableBarrierTs:457152680141520919 AdminJobType:noop} span: {tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}, resolvedTs: 457152680141520919, checkpointTs: 457152680141520919, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/292c6474-4b4b-4125-a38d-7f6efa3cd97a {"id":"292c6474-4b4b-4125-a38d-7f6efa3cd97a","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899112} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087bdb39f5 292c6474-4b4b-4125-a38d-7f6efa3cd97a /tidb/cdc/default/default/changefeed/info/9d7077c4-d482-42fd-b85e-bd9b18f0d9ec {"upstream-id":7489989421666590001,"namespace":"default","changefeed-id":"9d7077c4-d482-42fd-b85e-bd9b18f0d9ec","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2025-04-06T08:24:31.34006381+08:00","start-ts":457152678123012101,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457152678149226502} /tidb/cdc/default/default/changefeed/status/9d7077c4-d482-42fd-b85e-bd9b18f0d9ec {"checkpoint-ts":457152680141520919,"min-table-barrier-ts":457152680141520919,"admin-job-type":0} /tidb/cdc/default/default/task/position/292c6474-4b4b-4125-a38d-7f6efa3cd97a/9d7077c4-d482-42fd-b85e-bd9b18f0d9ec {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489989421666590001 {"id":7489989421666590001,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x check_capture_count http://127.0.0.1:2379 1 run task successfully wait process cdc.test exit for 2-th time... + set +x + run_sql 'USE TEST;Create table t1(a int primary key, b int);insert into t1 values(1,2);insert into t1 values(2,3);' [2025/04/06 08:25:15.005 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 368, b: 368, c: o, ahRhZ: VgauzvAfkJ, d: 08:30:54, }"] [row2="{ a: 368, b: 368, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:30:54, }"] [2025/04/06 08:25:15.105 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ b: 369, c: o, ahRhZ: VgauzvAfkJ, d: 08:30:55, a: 369, }"] [row2="{ a: 369, b: 369, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:30:55, }"] [2025/04/06 08:25:15.205 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 370, b: 370, c: o, ahRhZ: VgauzvAfkJ, d: 08:30:56, }"] [row2="{ a: 370, b: 370, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:30:56, }"] nonempty select id, val from test.availability1 where id=3 and val=3 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later wait process cdc.test exit for 1-th time... [2025/04/06 08:25:15.305 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 371, b: 371, c: o, ahRhZ: VgauzvAfkJ, d: 08:30:57, }"] [row2="{ d: 08:30:57, a: 371, b: 371, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, }"] [2025/04/06 08:25:15.406 +08:00] [WARN] [diff.go:937] ["find different row"] [column=ahRhZ] [row1="{ c: o, ahRhZ: VgauzvAfkJ, d: 08:30:58, a: 372, b: 372, }"] [row2="{ ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:30:58, a: 372, b: 372, c: D, }"] wait process cdc.test exit for 3-th time... + check_table_exists test.t1 127.0.0.1 3306 table test.t1 not exists for 1-th check, retry later [2025/04/06 08:25:15.506 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 373, b: 373, c: o, ahRhZ: VgauzvAfkJ, d: 08:30:59, }"] [row2="{ a: 373, b: 373, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:30:59, }"] [2025/04/06 08:25:15.606 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 374, b: 374, c: o, ahRhZ: VgauzvAfkJ, d: 08:31:00, }"] [row2="{ d: 08:31:00, a: 374, b: 374, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, }"] [2025/04/06 08:25:15.706 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ c: o, ahRhZ: VgauzvAfkJ, d: 08:31:01, a: 375, b: 375, }"] [row2="{ a: 375, b: 375, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:31:01, }"] wait process cdc.test exit for 2-th time... [2025/04/06 08:25:15.806 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ b: 376, c: o, ahRhZ: VgauzvAfkJ, d: 08:31:02, a: 376, }"] [row2="{ a: 376, b: 376, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:31:02, }"] [2025/04/06 08:25:15.907 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 377, b: 377, c: o, ahRhZ: VgauzvAfkJ, d: 08:31:03, }"] [row2="{ a: 377, b: 377, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:31:03, }"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Sun Apr 6 08:25:16 CST 2025] <<<<<< START cdc server in consistent_replicate_storage_file case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sink/dmlsink/txn/mysql/MySQLSinkHangLongTime=return(true)' + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.consistent_replicate_storage_file.1153111533.out server --log-file /tmp/tidb_cdc_test/consistent_replicate_storage_file/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/consistent_replicate_storage_file/cdc_data --cluster-id default + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2025/04/06 08:25:16.007 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 378, b: 378, c: o, ahRhZ: VgauzvAfkJ, d: 08:31:04, }"] [row2="{ ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:31:04, a: 378, b: 378, c: D, }"] [2025/04/06 08:25:16.107 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ ahRhZ: VgauzvAfkJ, d: 08:31:05, a: 379, b: 379, c: o, }"] [row2="{ a: 379, b: 379, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:31:05, }"] [2025/04/06 08:25:16.207 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ c: o, ahRhZ: VgauzvAfkJ, d: 08:31:07, a: 380, b: 380, }"] [row2="{ b: 380, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:31:07, a: 380, }"] wait process cdc.test exit for 3-th time... [2025/04/06 08:25:16.307 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ ahRhZ: VgauzvAfkJ, d: 08:31:08, a: 381, b: 381, c: o, }"] [row2="{ d: 08:31:08, a: 381, b: 381, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, }"] [2025/04/06 08:25:16.408 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 382, b: 382, c: o, ahRhZ: VgauzvAfkJ, d: 08:31:09, }"] [row2="{ c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:31:09, a: 382, b: 382, }"] [Sun Apr 6 08:25:16 CST 2025] <<<<<< START cdc server in multi_rocks case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + GO_FAILPOINTS= + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_rocks.2742427426.out server --log-file /tmp/tidb_cdc_test/multi_rocks/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_rocks/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Sun Apr 6 08:25:16 CST 2025] <<<<<< run test case ddl_reentrant success! >>>>>> [2025/04/06 08:25:16.508 +08:00] [WARN] [diff.go:937] ["find different row"] [column=ahRhZ] [row1="{ a: 383, b: 383, c: o, ahRhZ: VgauzvAfkJ, d: 08:31:10, }"] [row2="{ d: 08:31:10, a: 383, b: 383, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, }"] [2025/04/06 08:25:16.608 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 384, b: 384, c: o, ahRhZ: VgauzvAfkJ, d: 08:31:11, }"] [row2="{ c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:31:11, a: 384, b: 384, }"] [2025/04/06 08:25:16.708 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ b: 385, c: o, ahRhZ: VgauzvAfkJ, d: 08:31:12, a: 385, }"] [row2="{ a: 385, b: 385, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:31:12, }"] [2025/04/06 08:25:16.808 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ ahRhZ: VgauzvAfkJ, d: 08:31:13, a: 386, b: 386, c: o, }"] [row2="{ a: 386, b: 386, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:31:13, }"] [2025/04/06 08:25:16.908 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 387, b: 387, c: o, ahRhZ: VgauzvAfkJ, d: 08:31:14, }"] [row2="{ ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:31:14, a: 387, b: 387, c: D, }"] Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release [2025/04/06 08:25:17.009 +08:00] [WARN] [diff.go:937] ["find different row"] [column=ahRhZ] [row1="{ c: o, ahRhZ: VgauzvAfkJ, d: 08:31:15, a: 388, b: 388, }"] [row2="{ d: 08:31:15, a: 388, b: 388, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, }"] [2025/04/06 08:25:17.109 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ b: 389, c: o, ahRhZ: VgauzvAfkJ, d: 08:31:16, a: 389, }"] [row2="{ d: 08:31:16, a: 389, b: 389, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, }"] [2025/04/06 08:25:17.209 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 390, b: 390, c: o, ahRhZ: VgauzvAfkJ, d: 08:31:17, }"] [row2="{ c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:31:17, a: 390, b: 390, }"] [2025/04/06 08:25:17.309 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ d: 08:31:18, a: 391, b: 391, c: o, ahRhZ: VgauzvAfkJ, }"] [row2="{ b: 391, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:31:18, a: 391, }"] [2025/04/06 08:25:17.410 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 392, b: 392, c: o, ahRhZ: VgauzvAfkJ, d: 08:31:19, }"] [row2="{ a: 392, b: 392, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:31:19, }"] Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release [2025/04/06 08:25:17.510 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ c: o, ahRhZ: VgauzvAfkJ, d: 08:31:20, a: 393, b: 393, }"] [row2="{ a: 393, b: 393, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:31:20, }"] [2025/04/06 08:25:17.610 +08:00] [WARN] [diff.go:937] ["find different row"] [column=ahRhZ] [row1="{ a: 395, b: 395, c: o, ahRhZ: VgauzvAfkJ, d: 08:31:22, }"] [row2="{ ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:31:22, a: 395, b: 395, c: D, }"] [2025/04/06 08:25:17.710 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ c: o, ahRhZ: VgauzvAfkJ, d: 08:31:23, a: 396, b: 396, }"] [row2="{ d: 08:31:23, a: 396, b: 396, c: UcICuMitNk, ahRhZ: QdArKAP2RgOEVYU7Y, }"] nonempty select id, val from test.availability1 where id=3 and val=3 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 2-th time, retry later [2025/04/06 08:25:17.810 +08:00] [WARN] [diff.go:937] ["find different row"] [column=ahRhZ] [row1="{ b: 397, c: o, ahRhZ: VgauzvAfkJ, d: 08:31:24, a: 397, }"] [row2="{ a: 397, b: 397, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:31:24, }"] [2025/04/06 08:25:17.911 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 398, b: 398, c: o, ahRhZ: VgauzvAfkJ, d: 08:31:25, }"] [row2="{ a: 398, b: 398, c: D, ahRhZ: 7ioVcUnHC0J7rZqnFM38a, d: 08:31:25, }"] table test.t1 exists + sleep 5 [2025/04/06 08:25:18.011 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 399, b: 399, c: o, ahRhZ: VgauzvAfkJ, d: 08:31:26, }"] [row2="{ d: 08:31:26, a: 399, b: 399, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, }"] [2025/04/06 08:25:18.111 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 400, b: 400, c: o, ahRhZ: VgauzvAfkJ, d: 08:31:27, }"] [row2="{ a: 400, b: 400, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:31:27, }"] [2025/04/06 08:25:18.211 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ ahRhZ: VgauzvAfkJ, d: 08:31:28, a: 401, b: 401, c: o, }"] [row2="{ d: 08:31:28, a: 401, b: 401, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, }"] changefeed count 0 check pass, pd_addr: http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_with_auth.cli.12242.out cli changefeed create --sink-uri=mysql://normal:123456@127.0.0.1:3306/ --tz=Asia/Shanghai -c=custom-changefeed-name VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489989647983861866 TiDB Cluster ID. tikv_gc_leader_uuid 65821f30ca40004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-c-m09xf, pid:9978, start at 2025-04-06 08:25:16.335414791 +0800 CST m=+2.442247238 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:27:16.343 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:25:16.329 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:15:16.329 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489989647983861866 TiDB Cluster ID. tikv_gc_leader_uuid 65821f30ca40004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-c-m09xf, pid:9978, start at 2025-04-06 08:25:16.335414791 +0800 CST m=+2.442247238 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:27:16.343 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:25:16.329 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:15:16.329 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489989640318219964 TiDB Cluster ID. tikv_gc_leader_uuid 65821f30e34000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-c-m09xf, pid:10057, start at 2025-04-06 08:25:16.447607294 +0800 CST m=+2.478911866 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:27:16.460 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:25:16.429 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:15:16.429 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/changefeed_auto_stop/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/changefeed_auto_stop/tiflash/log/error.log arg matches is ArgMatches { args: {"log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/changefeed_auto_stop/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/changefeed_auto_stop/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/changefeed_auto_stop/tiflash/db/proxy"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [2025/04/06 08:25:18.311 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 402, b: 402, c: o, ahRhZ: VgauzvAfkJ, d: 08:31:29, }"] [row2="{ a: 402, b: 402, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:31:29, }"] [2025/04/06 08:25:18.411 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 403, b: 403, c: o, ahRhZ: VgauzvAfkJ, d: 08:31:30, }"] [row2="{ a: 403, b: 403, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:31:30, }"] [2025/04/06 08:25:18.512 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 404, b: 404, c: o, ahRhZ: VgauzvAfkJ, d: 08:31:31, }"] [row2="{ a: 404, b: 404, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:31:31, }"] [WARN] --tz is deprecated in changefeed settings. Create changefeed successfully! ID: custom-changefeed-name Info: {"upstream_id":7489989501405550986,"namespace":"default","id":"custom-changefeed-name","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-06T08:25:18.409620953+08:00","start_ts":457152690465013764,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457152690465013764,"checkpoint_ts":457152690465013764,"checkpoint_time":"2025-04-06 08:25:18.290"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... [2025/04/06 08:25:18.612 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ c: o, ahRhZ: VgauzvAfkJ, d: 08:31:32, a: 405, b: 405, }"] [row2="{ c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:31:32, a: 405, b: 405, }"] [2025/04/06 08:25:18.712 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 406, b: 406, c: o, ahRhZ: VgauzvAfkJ, d: 08:31:33, }"] [row2="{ a: 406, b: 406, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:31:33, }"] Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2025/04/06 08:25:18.812 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 407, b: 407, c: o, ahRhZ: VgauzvAfkJ, d: 08:31:34, }"] [row2="{ d: 08:31:34, a: 407, b: 407, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, }"] [2025/04/06 08:25:18.912 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 408, b: 408, c: o, ahRhZ: VgauzvAfkJ, d: 08:31:35, }"] [row2="{ a: 408, b: 408, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:31:35, }"] [2025/04/06 08:25:19.013 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 409, b: 409, c: o, ahRhZ: VgauzvAfkJ, d: 08:31:36, }"] [row2="{ a: 409, b: 409, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:31:36, }"] [2025/04/06 08:25:19.113 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ c: o, ahRhZ: VgauzvAfkJ, d: 08:31:37, a: 410, b: 410, }"] [row2="{ c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:31:37, a: 410, b: 410, }"] [2025/04/06 08:25:19.213 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 411, b: 411, c: o, ahRhZ: VgauzvAfkJ, d: 08:31:38, }"] [row2="{ b: 411, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:31:38, a: 411, }"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 06 Apr 2025 00:25:19 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/cb6b1b7f-ab29-4852-a097-f142c1ff4ff0 {UpstreamID:7489989380902714499 Namespace:default ID:cb6b1b7f-ab29-4852-a097-f142c1ff4ff0 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-06 08:24:21.282277133 +0800 CST StartTs:457152675499212804 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00397a360 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457152675512057862} {CheckpointTs:457152690375622660 MinTableBarrierTs:457152690375622662 AdminJobType:noop} span: {tableID:123, startKey:7480000000000000ff7b5f720000000000fa, endKey:7480000000000000ff7b5f730000000000fa}, resolvedTs: 457152690375622662, checkpointTs: 457152690375622660, state: Replicating span: {tableID:124, startKey:7480000000000000ff7c5f720000000000fa, endKey:7480000000000000ff7c5f730000000000fa}, resolvedTs: 457152690375622660, checkpointTs: 457152690375622660, state: Replicating span: {tableID:121, startKey:7480000000000000ff795f720000000000fa, endKey:7480000000000000ff795f730000000000fa}, resolvedTs: 457152690375622660, checkpointTs: 457152690375622660, state: Replicating span: {tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}, resolvedTs: 457152690375622662, checkpointTs: 457152690375622660, state: Replicating span: {tableID:116, startKey:7480000000000000ff745f720000000000fa, endKey:7480000000000000ff745f730000000000fa}, resolvedTs: 457152690375622662, checkpointTs: 457152690375622660, state: Replicating span: {tableID:119, startKey:7480000000000000ff775f720000000000fa, endKey:7480000000000000ff775f730000000000fa}, resolvedTs: 457152690375622661, checkpointTs: 457152690375622660, state: Replicating span: {tableID:120, startKey:7480000000000000ff785f720000000000fa, endKey:7480000000000000ff785f730000000000fa}, resolvedTs: 457152690375622660, checkpointTs: 457152690375622660, state: Replicating span: {tableID:126, startKey:7480000000000000ff7e5f720000000000fa, endKey:7480000000000000ff7e5f730000000000fa}, resolvedTs: 457152690375622662, checkpointTs: 457152690375622660, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/247d0fe5-b75b-4a7f-af33-a4097f587134 {"id":"247d0fe5-b75b-4a7f-af33-a4097f587134","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899116} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087bb09e18 247d0fe5-b75b-4a7f-af33-a4097f587134 /tidb/cdc/default/default/changefeed/info/cb6b1b7f-ab29-4852-a097-f142c1ff4ff0 {"upstream-id":7489989380902714499,"namespace":"default","changefeed-id":"cb6b1b7f-ab29-4852-a097-f142c1ff4ff0","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-06T08:24:21.282277133+08:00","start-ts":457152675499212804,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"file:///tmp/tidb_cdc_test/consistent_replicate_storage_file/redo","use-file-backend":false,"compression":"lz4","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457152675512057862} /tidb/cdc/default/default/changefeed/status/cb6b1b7f-ab29-4852-a097-f142c1ff4ff0 {"checkpoint-ts":457152690375622660,"min-table-barrier-ts":457152690638028807,"admin-job-type":0} /tidb/cdc/default/default/task/position/247d0fe5-b75b-4a7f-af33-a4097f587134/cb6b1b7f-ab29-4852-a097-f142c1ff4ff0 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489989380902714499 {"id":7489989380902714499,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/cb6b1b7f-ab29-4852-a097-f142c1ff4ff0 {UpstreamID:7489989380902714499 Namespace:default ID:cb6b1b7f-ab29-4852-a097-f142c1ff4ff0 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-06 08:24:21.282277133 +0800 CST StartTs:457152675499212804 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00397a360 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457152675512057862} {CheckpointTs:457152690375622660 MinTableBarrierTs:457152690375622662 AdminJobType:noop} span: {tableID:123, startKey:7480000000000000ff7b5f720000000000fa, endKey:7480000000000000ff7b5f730000000000fa}, resolvedTs: 457152690375622662, checkpointTs: 457152690375622660, state: Replicating span: {tableID:124, startKey:7480000000000000ff7c5f720000000000fa, endKey:7480000000000000ff7c5f730000000000fa}, resolvedTs: 457152690375622660, checkpointTs: 457152690375622660, state: Replicating span: {tableID:121, startKey:7480000000000000ff795f720000000000fa, endKey:7480000000000000ff795f730000000000fa}, resolvedTs: 457152690375622660, checkpointTs: 457152690375622660, state: Replicating span: {tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}, resolvedTs: 457152690375622662, checkpointTs: 457152690375622660, state: Replicating span: {tableID:116, startKey:7480000000000000ff745f720000000000fa, endKey:7480000000000000ff745f730000000000fa}, resolvedTs: 457152690375622662, checkpointTs: 457152690375622660, state: Replicating span: {tableID:119, startKey:7480000000000000ff775f720000000000fa, endKey:7480000000000000ff775f730000000000fa}, resolvedTs: 457152690375622661, checkpointTs: 457152690375622660, state: Replicating span: {tableID:120, startKey:7480000000000000ff785f720000000000fa, endKey:7480000000000000ff785f730000000000fa}, resolvedTs: 457152690375622660, checkpointTs: 457152690375622660, state: Replicating span: {tableID:126, startKey:7480000000000000ff7e5f720000000000fa, endKey:7480000000000000ff7e5f730000000000fa}, resolvedTs: 457152690375622662, checkpointTs: 457152690375622660, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/247d0fe5-b75b-4a7f-af33-a4097f587134 {"id":"247d0fe5-b75b-4a7f-af33-a4097f587134","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899116} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087bb09e18 247d0fe5-b75b-4a7f-af33-a4097f587134 /tidb/cdc/default/default/changefeed/info/cb6b1b7f-ab29-4852-a097-f142c1ff4ff0 {"upstream-id":7489989380902714499,"namespace":"default","changefeed-id":"cb6b1b7f-ab29-4852-a097-f142c1ff4ff0","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-06T08:24:21.282277133+08:00","start-ts":457152675499212804,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"file:///tmp/tidb_cdc_test/consistent_replicate_storage_file/redo","use-file-backend":false,"compression":"lz4","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457152675512057862} /tidb/cdc/default/default/changefeed/status/cb6b1b7f-ab29-4852-a097-f142c1ff4ff0 {"checkpoint-ts":457152690375622660,"min-table-barrier-ts":457152690638028807,"admin-job-type":0} /tidb/cdc/default/default/task/position/247d0fe5-b75b-4a7f-af33-a4097f587134/cb6b1b7f-ab29-4852-a097-f142c1ff4ff0 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489989380902714499 {"id":7489989380902714499,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/cb6b1b7f-ab29-4852-a097-f142c1ff4ff0 {UpstreamID:7489989380902714499 Namespace:default ID:cb6b1b7f-ab29-4852-a097-f142c1ff4ff0 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-06 08:24:21.282277133 +0800 CST StartTs:457152675499212804 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00397a360 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457152675512057862} {CheckpointTs:457152690375622660 MinTableBarrierTs:457152690375622662 AdminJobType:noop} span: {tableID:123, startKey:7480000000000000ff7b5f720000000000fa, endKey:7480000000000000ff7b5f730000000000fa}, resolvedTs: 457152690375622662, checkpointTs: 457152690375622660, state: Replicating span: {tableID:124, startKey:7480000000000000ff7c5f720000000000fa, endKey:7480000000000000ff7c5f730000000000fa}, resolvedTs: 457152690375622660, checkpointTs: 457152690375622660, state: Replicating span: {tableID:121, startKey:7480000000000000ff795f720000000000fa, endKey:7480000000000000ff795f730000000000fa}, resolvedTs: 457152690375622660, checkpointTs: 457152690375622660, state: Replicating span: {tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}, resolvedTs: 457152690375622662, checkpointTs: 457152690375622660, state: Replicating span: {tableID:116, startKey:7480000000000000ff745f720000000000fa, endKey:7480000000000000ff745f730000000000fa}, resolvedTs: 457152690375622662, checkpointTs: 457152690375622660, state: Replicating span: {tableID:119, startKey:7480000000000000ff775f720000000000fa, endKey:7480000000000000ff775f730000000000fa}, resolvedTs: 457152690375622661, checkpointTs: 457152690375622660, state: Replicating span: {tableID:120, startKey:7480000000000000ff785f720000000000fa, endKey:7480000000000000ff785f730000000000fa}, resolvedTs: 457152690375622660, checkpointTs: 457152690375622660, state: Replicating span: {tableID:126, startKey:7480000000000000ff7e5f720000000000fa, endKey:7480000000000000ff7e5f730000000000fa}, resolvedTs: 457152690375622662, checkpointTs: 457152690375622660, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/247d0fe5-b75b-4a7f-af33-a4097f587134 {"id":"247d0fe5-b75b-4a7f-af33-a4097f587134","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899116} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087bb09e18 247d0fe5-b75b-4a7f-af33-a4097f587134 /tidb/cdc/default/default/changefeed/info/cb6b1b7f-ab29-4852-a097-f142c1ff4ff0 {"upstream-id":7489989380902714499,"namespace":"default","changefeed-id":"cb6b1b7f-ab29-4852-a097-f142c1ff4ff0","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-06T08:24:21.282277133+08:00","start-ts":457152675499212804,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"file:///tmp/tidb_cdc_test/consistent_replicate_storage_file/redo","use-file-backend":false,"compression":"lz4","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457152675512057862} /tidb/cdc/default/default/changefeed/status/cb6b1b7f-ab29-4852-a097-f142c1ff4ff0 {"checkpoint-ts":457152690375622660,"min-table-barrier-ts":457152690638028807,"admin-job-type":0} + grep -q 'etcd info' /tidb/cdc/default/default/task/position/247d0fe5-b75b-4a7f-af33-a4097f587134/cb6b1b7f-ab29-4852-a097-f142c1ff4ff0 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489989380902714499 {"id":7489989380902714499,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x [2025/04/06 08:25:19.313 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 412, b: 412, c: o, ahRhZ: VgauzvAfkJ, d: 08:31:39, }"] [row2="{ b: 412, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:31:39, a: 412, }"] [2025/04/06 08:25:19.413 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ ahRhZ: VgauzvAfkJ, d: 08:31:40, a: 413, b: 413, c: o, }"] [row2="{ ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:31:40, a: 413, b: 413, c: D, }"] [2025/04/06 08:25:19.514 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 414, b: 414, c: o, ahRhZ: VgauzvAfkJ, d: 08:31:41, }"] [row2="{ a: 414, b: 414, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:31:41, }"] [2025/04/06 08:25:19.614 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 416, b: 416, c: o, ahRhZ: VgauzvAfkJ, d: 08:31:43, }"] [row2="{ a: 416, b: 416, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:31:43, }"] [2025/04/06 08:25:19.714 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ ahRhZ: VgauzvAfkJ, d: 08:31:44, a: 417, b: 417, c: o, }"] [row2="{ c: dK, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:31:44, a: 417, b: 417, }"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 06 Apr 2025 00:25:19 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c7513e7e-b535-462c-8fa4-267bdcc6a774 {"id":"c7513e7e-b535-462c-8fa4-267bdcc6a774","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899116} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087c41d426 c7513e7e-b535-462c-8fa4-267bdcc6a774 /tidb/cdc/default/default/upstream/7489989538505028507 {"id":7489989538505028507,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c7513e7e-b535-462c-8fa4-267bdcc6a774 {"id":"c7513e7e-b535-462c-8fa4-267bdcc6a774","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899116} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087c41d426 c7513e7e-b535-462c-8fa4-267bdcc6a774 /tidb/cdc/default/default/upstream/7489989538505028507 {"id":7489989538505028507,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c7513e7e-b535-462c-8fa4-267bdcc6a774 {"id":"c7513e7e-b535-462c-8fa4-267bdcc6a774","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899116} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087c41d426 c7513e7e-b535-462c-8fa4-267bdcc6a774 /tidb/cdc/default/default/upstream/7489989538505028507 {"id":7489989538505028507,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_rocks.cli.27837.out cli changefeed create --sink-uri=mysql://normal:123456@127.0.0.1:3306/ Create changefeed successfully! ID: 45187425-ea8b-461c-8673-5bd85685edb0 Info: {"upstream_id":7489989538505028507,"namespace":"default","id":"45187425-ea8b-461c-8673-5bd85685edb0","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-06T08:25:19.828510923+08:00","start_ts":457152690841190402,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457152690841190402,"checkpoint_ts":457152690841190402,"checkpoint_time":"2025-04-06 08:25:19.725"} PASS + set +x [2025/04/06 08:25:19.814 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ ahRhZ: VgauzvAfkJ, d: 08:31:45, a: 418, b: 418, c: o, }"] [row2="{ a: 418, b: 418, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:31:45, }"] [2025/04/06 08:25:19.914 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ ahRhZ: VgauzvAfkJ, d: 08:31:46, a: 419, b: 419, c: o, }"] [row2="{ a: 419, b: 419, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:31:46, }"] [2025/04/06 08:25:20.015 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 420, b: 420, c: o, ahRhZ: VgauzvAfkJ, d: 08:31:47, }"] [row2="{ d: 08:31:47, a: 420, b: 420, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, }"] Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) coverage: 2.4% of statements in github.com/pingcap/tiflow/... [2025/04/06 08:25:20.115 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 421, b: 421, c: o, ahRhZ: VgauzvAfkJ, d: 08:35:47, }"] [row2="{ b: 421, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:35:47, a: 421, }"] [2025/04/06 08:25:20.215 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 422, b: 422, c: o, ahRhZ: VgauzvAfkJ, d: 08:31:49, }"] [row2="{ a: 422, b: 422, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:31:49, }"] [2025/04/06 08:25:20.315 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 423, b: 423, c: o, ahRhZ: VgauzvAfkJ, d: 08:31:50, }"] [row2="{ a: 423, b: 423, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:31:50, }"] [2025/04/06 08:25:20.415 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ b: 424, c: o, ahRhZ: VgauzvAfkJ, d: 08:31:51, a: 424, }"] [row2="{ c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:31:51, a: 424, b: 424, }"] [2025/04/06 08:25:20.515 +08:00] [WARN] [diff.go:937] ["find different row"] [column=ahRhZ] [row1="{ a: 425, b: 425, c: o, ahRhZ: VgauzvAfkJ, d: 08:31:52, }"] [row2="{ a: 425, b: 425, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:31:52, }"] [2025/04/06 08:25:20.616 +08:00] [WARN] [diff.go:937] ["find different row"] [column=ahRhZ] [row1="{ a: 426, b: 426, c: o, ahRhZ: VgauzvAfkJ, d: 08:31:53, }"] [row2="{ ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:31:53, a: 426, b: 426, c: D, }"] [2025/04/06 08:25:20.716 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ b: 427, c: o, ahRhZ: VgauzvAfkJ, d: 08:31:54, a: 427, }"] [row2="{ a: 427, b: 427, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:31:54, }"] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489989357300557951 TiDB Cluster ID. tikv_gc_leader_uuid 65821ef08600007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-j-shst4, pid:9157, start at 2025-04-06 08:24:10.546273365 +0800 CST m=+3.817043453 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:26:10.556 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:24:10.520 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:14:10.520 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489989357300557951 TiDB Cluster ID. tikv_gc_leader_uuid 65821ef08600007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-j-shst4, pid:9157, start at 2025-04-06 08:24:10.546273365 +0800 CST m=+3.817043453 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:26:10.556 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:24:10.520 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:14:10.520 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489989362460687017 TiDB Cluster ID. tikv_gc_leader_uuid 65821ef09fc0002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-j-shst4, pid:9225, start at 2025-04-06 08:24:10.627926606 +0800 CST m=+3.832542938 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:26:10.644 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:24:10.623 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:14:10.623 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... ***************** properties ***************** "insertproportion"="0" "threadcount"="4" "readproportion"="0" "workload"="core" "operationcount"="0" "updateproportion"="0" "dotransactions"="false" "recordcount"="20" "mysql.host"="127.0.0.1" "requestdistribution"="uniform" "mysql.port"="4000" "mysql.user"="root" "mysql.db"="changefeed_auto_stop_1" "scanproportion"="0" "readallfields"="true" ********************************************** Run finished, takes 10.827539ms INSERT - Takes(s): 0.0, Count: 20, OPS: 3237.1, Avg(us): 2046, Min(us): 1243, Max(us): 4656, 95th(us): 5000, 99th(us): 5000 ***************** properties ***************** "insertproportion"="0" "readallfields"="true" "mysql.db"="changefeed_auto_stop_2" "operationcount"="0" "requestdistribution"="uniform" "updateproportion"="0" "dotransactions"="false" "threadcount"="4" "mysql.port"="4000" "workload"="core" "mysql.host"="127.0.0.1" "mysql.user"="root" "scanproportion"="0" "readproportion"="0" "recordcount"="20" ********************************************** Run finished, takes 11.597726ms INSERT - Takes(s): 0.0, Count: 20, OPS: 3001.9, Avg(us): 2112, Min(us): 1190, Max(us): 5282, 95th(us): 5000, 99th(us): 6000 ***************** properties ***************** "mysql.host"="127.0.0.1" "mysql.user"="root" "insertproportion"="0" "readproportion"="0" "dotransactions"="false" "requestdistribution"="uniform" "readallfields"="true" "updateproportion"="0" "operationcount"="0" "mysql.port"="4000" "threadcount"="4" "scanproportion"="0" "workload"="core" "mysql.db"="changefeed_auto_stop_3" "recordcount"="20" ********************************************** Run finished, takes 12.351625ms INSERT - Takes(s): 0.0, Count: 20, OPS: 2656.0, Avg(us): 2275, Min(us): 1219, Max(us): 4845, 95th(us): 5000, 99th(us): 5000 ***************** properties ***************** "mysql.host"="127.0.0.1" "workload"="core" "readallfields"="true" "operationcount"="0" "requestdistribution"="uniform" "mysql.user"="root" "dotransactions"="false" "threadcount"="4" "updateproportion"="0" "mysql.port"="4000" "recordcount"="20" "insertproportion"="0" "mysql.db"="changefeed_auto_stop_4" "scanproportion"="0" "readproportion"="0" ********************************************** Run finished, takes 11.270171ms INSERT - Takes(s): 0.0, Count: 20, OPS: 2826.6, Avg(us): 2093, Min(us): 1230, Max(us): 4083, 95th(us): 5000, 99th(us): 5000 [Sun Apr 6 08:25:20 CST 2025] <<<<<< START cdc server in changefeed_auto_stop case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_auto_stop.1148211484.out server --log-file /tmp/tidb_cdc_test/changefeed_auto_stop/cdc1.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_auto_stop/cdc_data1 --cluster-id default --addr 127.0.0.1:8301 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2025/04/06 08:25:20.816 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ ahRhZ: VgauzvAfkJ, d: 08:31:55, a: 428, b: 428, c: o, }"] [row2="{ a: 428, b: 428, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:31:55, }"] [2025/04/06 08:25:20.916 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 429, b: 429, c: o, ahRhZ: VgauzvAfkJ, d: 08:31:56, }"] [row2="{ a: 429, b: 429, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:31:56, }"] [2025/04/06 08:25:21.016 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ b: 430, c: o, ahRhZ: VgauzvAfkJ, d: 08:31:58, a: 430, }"] [row2="{ a: 430, b: 430, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:31:58, }"] [2025/04/06 08:25:21.117 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 431, b: 431, c: o, ahRhZ: VgauzvAfkJ, d: 08:31:59, }"] [row2="{ ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:31:59, a: 431, b: 431, c: D, }"] [2025/04/06 08:25:21.217 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 432, b: 432, c: o, ahRhZ: VgauzvAfkJ, d: 08:32:00, }"] [row2="{ ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:32:00, a: 432, b: 432, c: Rx53, }"] + set +x [2025/04/06 08:25:21.317 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 433, b: 433, c: o, ahRhZ: VgauzvAfkJ, d: 08:32:01, }"] [row2="{ a: 433, b: 433, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:32:01, }"] [2025/04/06 08:25:21.417 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ ahRhZ: VgauzvAfkJ, d: 08:32:02, a: 434, b: 434, c: o, }"] [row2="{ d: 08:32:02, a: 434, b: 434, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, }"] [2025/04/06 08:25:21.517 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ b: 435, c: o, ahRhZ: VgauzvAfkJ, d: 08:32:03, a: 435, }"] [row2="{ c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:32:03, a: 435, b: 435, }"] [2025/04/06 08:25:21.618 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 436, b: 436, c: o, ahRhZ: VgauzvAfkJ, d: 08:32:04, }"] [row2="{ b: 436, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:32:04, a: 436, }"] [2025/04/06 08:25:21.718 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 437, b: 437, c: o, ahRhZ: VgauzvAfkJ, d: 08:32:05, }"] [row2="{ a: 437, b: 437, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:32:05, }"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) nonempty select id, val from test.availability1 where id=3 and val=3 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 3-th time, retry later [2025/04/06 08:25:21.818 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ ahRhZ: VgauzvAfkJ, d: 08:32:06, a: 438, b: 438, c: o, }"] [row2="{ b: 438, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:32:06, a: 438, }"] [2025/04/06 08:25:21.918 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 439, b: 439, c: o, ahRhZ: VgauzvAfkJ, d: 08:32:07, }"] [row2="{ a: 439, b: 439, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:32:07, }"] [2025/04/06 08:25:22.018 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ b: 440, c: o, ahRhZ: VgauzvAfkJ, d: 08:32:08, a: 440, }"] [row2="{ c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:32:08, a: 440, b: 440, }"] [2025/04/06 08:25:22.119 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ ahRhZ: VgauzvAfkJ, d: 08:32:09, a: 441, b: 441, c: o, }"] [row2="{ a: 441, b: 441, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:32:09, }"] [2025/04/06 08:25:22.219 +08:00] [WARN] [diff.go:937] ["find different row"] [column=ahRhZ] [row1="{ d: 08:32:10, a: 442, b: 442, c: o, ahRhZ: VgauzvAfkJ, }"] [row2="{ ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:32:10, a: 442, b: 442, c: D, }"] [2025/04/06 08:25:22.319 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 444, b: 444, c: o, ahRhZ: VgauzvAfkJ, d: 08:32:12, }"] [row2="{ a: 444, b: 444, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:32:12, }"] ***************** properties ***************** "table"="a1" "readallfields"="true" "operationcount"="0" "updateproportion"="0" "requestdistribution"="uniform" "mysql.host"="127.0.0.1" "recordcount"="1000" "readproportion"="0" "mysql.port"="4000" "mysql.db"="multi_rocks" "insertproportion"="0" "threadcount"="2" "dotransactions"="false" "mysql.user"="root" "scanproportion"="0" "workload"="core" ********************************************** [2025/04/06 08:25:22.419 +08:00] [WARN] [diff.go:937] ["find different row"] [column=ahRhZ] [row1="{ ahRhZ: VgauzvAfkJ, d: 08:32:13, a: 445, b: 445, c: o, }"] [row2="{ ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:32:13, a: 445, b: 445, c: D, }"] [2025/04/06 08:25:22.519 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 446, b: 446, c: o, ahRhZ: VgauzvAfkJ, d: 08:32:14, }"] [row2="{ a: 446, b: 446, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:32:14, }"] [2025/04/06 08:25:22.620 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 447, b: 447, c: o, ahRhZ: VgauzvAfkJ, d: 08:32:15, }"] [row2="{ b: 447, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:32:15, a: 447, }"] [2025/04/06 08:25:22.720 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 448, b: 448, c: o, ahRhZ: VgauzvAfkJ, d: 08:32:16, }"] [row2="{ a: 448, b: 448, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:32:16, }"] [2025/04/06 08:25:22.820 +08:00] [WARN] [diff.go:937] ["find different row"] [column=ahRhZ] [row1="{ a: 449, b: 449, c: o, ahRhZ: VgauzvAfkJ, d: 08:32:17, }"] [row2="{ b: 449, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:32:17, a: 449, }"] Run finished, takes 620.585215ms INSERT - Takes(s): 0.6, Count: 999, OPS: 1654.9, Avg(us): 1193, Min(us): 910, Max(us): 16815, 95th(us): 2000, 99th(us): 2000 [2025/04/06 08:25:22.920 +08:00] [WARN] [diff.go:937] ["find different row"] [column=ahRhZ] [row1="{ c: o, ahRhZ: VgauzvAfkJ, d: 08:32:18, a: 450, b: 450, }"] [row2="{ a: 450, b: 450, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:32:18, }"] [2025/04/06 08:25:23.020 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 451, b: 451, c: o, ahRhZ: VgauzvAfkJ, d: 08:32:19, }"] [row2="{ b: 451, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:32:19, a: 451, }"] + endpoints=http://127.0.0.1:2379 + changefeed_id=custom-changefeed-name + expected_state=normal + error_msg=null + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c custom-changefeed-name -s + info='{ "upstream_id": 7489989501405550986, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 457152691487375363, "checkpoint_time": "2025-04-06 08:25:22.190", "error": null }' + echo '{ "upstream_id": 7489989501405550986, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 457152691487375363, "checkpoint_time": "2025-04-06 08:25:22.190", "error": null }' { "upstream_id": 7489989501405550986, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 457152691487375363, "checkpoint_time": "2025-04-06 08:25:22.190", "error": null } ++ echo '{' '"upstream_id":' 7489989501405550986, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 457152691487375363, '"checkpoint_time":' '"2025-04-06' '08:25:22.190",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7489989501405550986, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 457152691487375363, '"checkpoint_time":' '"2025-04-06' '08:25:22.190",' '"error":' null '}' ++ jq -r .error.message [2025/04/06 08:25:23.120 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ c: o, ahRhZ: VgauzvAfkJ, d: 08:32:20, a: 452, b: 452, }"] [row2="{ b: 452, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:32:20, a: 452, }"] [2025/04/06 08:25:23.221 +08:00] [WARN] [diff.go:937] ["find different row"] [column=ahRhZ] [row1="{ a: 453, b: 453, c: o, ahRhZ: VgauzvAfkJ, d: 08:32:21, }"] [row2="{ a: 453, b: 453, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:32:21, }"] [2025/04/06 08:25:23.321 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 454, b: 454, c: o, ahRhZ: VgauzvAfkJ, d: 08:32:22, }"] [row2="{ c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:32:22, a: 454, b: 454, }"] + message=null + [[ ! null =~ null ]] + kill_tikv ++ ps aux ++ grep tikv-server ++ grep /tmp/tidb_cdc_test/synced_status_with_redo + info='jenkins 12052 25.0 0.6 4848492 2491272 ? Sl 08:24 0:05 tikv-server --pd 127.0.0.1:2379 -A 127.0.0.1:20160 --status-addr 127.0.0.1:20181 --log-file /tmp/tidb_cdc_test/synced_status_with_redo/tikv1.log --log-level debug -C /tmp/tidb_cdc_test/synced_status_with_redo/tikv-config.toml -s /tmp/tidb_cdc_test/synced_status_with_redo/tikv1 jenkins 12053 35.6 0.6 4888428 2541404 ? Sl 08:24 0:08 tikv-server --pd 127.0.0.1:2379 -A 127.0.0.1:20161 --status-addr 127.0.0.1:20182 --log-file /tmp/tidb_cdc_test/synced_status_with_redo/tikv2.log --log-level debug -C /tmp/tidb_cdc_test/synced_status_with_redo/tikv-config.toml -s /tmp/tidb_cdc_test/synced_status_with_redo/tikv2 jenkins 12054 25.0 0.6 4851052 2486004 ? Sl 08:24 0:05 tikv-server --pd 127.0.0.1:2379 -A 127.0.0.1:20162 --status-addr 127.0.0.1:20183 --log-file /tmp/tidb_cdc_test/synced_status_with_redo/tikv3.log --log-level debug -C /tmp/tidb_cdc_test/synced_status_with_redo/tikv-config.toml -s /tmp/tidb_cdc_test/synced_status_with_redo/tikv3 jenkins 12056 34.1 0.6 4871532 2527400 ? Sl 08:24 0:07 tikv-server --pd 127.0.0.1:2479 -A 127.0.0.1:21160 --status-addr 127.0.0.1:21180 --log-file /tmp/tidb_cdc_test/synced_status_with_redo/tikv_down.log --log-level debug -C /tmp/tidb_cdc_test/synced_status_with_redo/tikv-config.toml -s /tmp/tidb_cdc_test/synced_status_with_redo/tikv_down' ++ ps aux ++ grep tikv-server ++ grep /tmp/tidb_cdc_test/synced_status_with_redo ++ awk '{print $2}' ++ xargs kill -9 ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 243 100 243 0 0 23525 0 --:--:-- --:--:-- --:--:-- 24300 + synced_status='{"synced":false,"sink_checkpoint_ts":"2025-04-06 08:25:20.918","puller_resolved_ts":"2025-04-06 08:25:15.118","last_synced_ts":"2025-04-06 08:25:15.618","now_ts":"2025-04-06 08:25:22.000","info":"The data syncing is not finished, please wait"}' ++ echo '{"synced":false,"sink_checkpoint_ts":"2025-04-06' '08:25:20.918","puller_resolved_ts":"2025-04-06' '08:25:15.118","last_synced_ts":"2025-04-06' '08:25:15.618","now_ts":"2025-04-06' '08:25:22.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq .synced + status=false + '[' false '!=' false ']' ++ echo '{"synced":false,"sink_checkpoint_ts":"2025-04-06' '08:25:20.918","puller_resolved_ts":"2025-04-06' '08:25:15.118","last_synced_ts":"2025-04-06' '08:25:15.618","now_ts":"2025-04-06' '08:25:22.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq -r .info + info='The data syncing is not finished, please wait' + target_message='The data syncing is not finished, please wait' + '[' 'The data syncing is not finished, please wait' '!=' 'The data syncing is not finished, please wait' ']' + sleep 130 ***************** properties ***************** "mysql.user"="root" "threadcount"="2" "readallfields"="true" "table"="a2" "dotransactions"="false" "updateproportion"="0" "insertproportion"="0" "operationcount"="0" "mysql.host"="127.0.0.1" "requestdistribution"="uniform" "recordcount"="1000" "mysql.db"="multi_rocks" "mysql.port"="4000" "workload"="core" "readproportion"="0" "scanproportion"="0" ********************************************** [2025/04/06 08:25:23.421 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ b: 455, c: o, ahRhZ: VgauzvAfkJ, d: 08:32:23, a: 455, }"] [row2="{ a: 455, b: 455, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:32:23, }"] [2025/04/06 08:25:23.522 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ ahRhZ: VgauzvAfkJ, d: 08:32:26, a: 458, b: 458, c: o, }"] [row2="{ a: 458, b: 458, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:32:26, }"] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489989674838134457 TiDB Cluster ID. tikv_gc_leader_uuid 65821f364b00009 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-j-vgmw9, pid:22120, start at 2025-04-06 08:25:21.976541899 +0800 CST m=+2.452096096 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:27:21.985 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:25:21.964 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:15:21.964 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489989674838134457 TiDB Cluster ID. tikv_gc_leader_uuid 65821f364b00009 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-j-vgmw9, pid:22120, start at 2025-04-06 08:25:21.976541899 +0800 CST m=+2.452096096 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:27:21.985 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:25:21.964 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:15:21.964 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489989674705300848 TiDB Cluster ID. tikv_gc_leader_uuid 65821f3673c000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-j-vgmw9, pid:22182, start at 2025-04-06 08:25:22.158361672 +0800 CST m=+2.565689730 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:27:22.167 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:25:22.161 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:15:22.161 +0800 All versions after safe point can be accessed. (DO NOT EDIT) [2025/04/06 08:25:23.622 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ d: 08:32:27, a: 459, b: 459, c: o, ahRhZ: VgauzvAfkJ, }"] [row2="{ d: 08:32:27, a: 459, b: 459, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, }"] [2025/04/06 08:25:23.722 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 460, b: 460, c: o, ahRhZ: VgauzvAfkJ, d: 08:32:28, }"] [row2="{ c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:32:28, a: 460, b: 460, }"] [2025/04/06 08:25:23.822 +08:00] [WARN] [diff.go:937] ["find different row"] [column=ahRhZ] [row1="{ a: 461, b: 461, c: o, ahRhZ: VgauzvAfkJ, d: 08:32:29, }"] [row2="{ a: 461, b: 461, c: 0bQ, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:32:29, }"] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_with_auth.cli.12338.out cli unsafe delete-service-gc-safepoint Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/move_table/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/move_table/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/move_table/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/move_table/tiflash-proxy.toml"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/move_table/tiflash/log/proxy.log"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [2025/04/06 08:25:23.922 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 462, b: 462, c: o, ahRhZ: VgauzvAfkJ, d: 08:32:30, }"] [row2="{ a: 462, b: 462, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:32:30, }"] [2025/04/06 08:25:24.023 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ d: 08:32:32, a: 464, b: 464, c: o, ahRhZ: VgauzvAfkJ, }"] [row2="{ a: 464, b: 464, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:32:32, }"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 06 Apr 2025 00:25:23 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4047b554-f313-4419-88c5-8530952b5e45 {"id":"4047b554-f313-4419-88c5-8530952b5e45","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899121} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087ca1bc15 4047b554-f313-4419-88c5-8530952b5e45 /tidb/cdc/default/default/upstream/7489989647983861866 {"id":7489989647983861866,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4047b554-f313-4419-88c5-8530952b5e45 {"id":"4047b554-f313-4419-88c5-8530952b5e45","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899121} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087ca1bc15 4047b554-f313-4419-88c5-8530952b5e45 /tidb/cdc/default/default/upstream/7489989647983861866 {"id":7489989647983861866,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4047b554-f313-4419-88c5-8530952b5e45 {"id":"4047b554-f313-4419-88c5-8530952b5e45","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899121} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087ca1bc15 4047b554-f313-4419-88c5-8530952b5e45 /tidb/cdc/default/default/upstream/7489989647983861866 {"id":7489989647983861866,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Sun Apr 6 08:25:23 CST 2025] <<<<<< START cdc server in changefeed_auto_stop case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/processor/pipeline/ProcessorSyncResolvedError=1*return(true);github.com/pingcap/tiflow/cdc/processor/ProcessorUpdatePositionDelaying=sleep(1000)' + [[ no != \n\o ]] + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_auto_stop.1154011542.out server --log-file /tmp/tidb_cdc_test/changefeed_auto_stop/cdc2.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_auto_stop/cdc_data2 --cluster-id default --addr 127.0.0.1:8302 --pd http://127.0.0.1:2379 + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8302; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2025/04/06 08:25:24.123 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ ahRhZ: VgauzvAfkJ, d: 08:32:33, a: 465, b: 465, c: o, }"] [row2="{ a: 465, b: 465, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:32:33, }"] [2025/04/06 08:25:24.223 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ c: o, ahRhZ: VgauzvAfkJ, d: 08:32:34, a: 466, b: 466, }"] [row2="{ a: 466, b: 466, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:32:34, }"] [2025/04/06 08:25:24.323 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 467, b: 467, c: o, ahRhZ: VgauzvAfkJ, d: 08:32:35, }"] [row2="{ a: 467, b: 467, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:32:35, }"] Run finished, takes 705.538217ms INSERT - Takes(s): 0.7, Count: 1000, OPS: 1466.6, Avg(us): 1369, Min(us): 962, Max(us): 23637, 95th(us): 2000, 99th(us): 3000 Confirm that you know what this command will do and use it at your own risk [Y/N] CDC service GC safepoint truncated in PD! PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... [2025/04/06 08:25:24.423 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 468, b: 468, c: o, ahRhZ: VgauzvAfkJ, d: 08:32:36, }"] [row2="{ a: 468, b: 468, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:32:36, }"] [2025/04/06 08:25:24.524 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ b: 470, c: o, ahRhZ: VgauzvAfkJ, d: 08:32:40, a: 470, }"] [row2="{ a: 470, b: 470, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:32:40, }"] [2025/04/06 08:25:24.624 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ c: o, ahRhZ: VgauzvAfkJ, d: 08:32:41, a: 471, b: 471, }"] [row2="{ b: 471, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:32:41, a: 471, }"] [2025/04/06 08:25:24.724 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 472, b: 472, c: o, ahRhZ: VgauzvAfkJ, d: 08:32:42, }"] [row2="{ c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:32:42, a: 472, b: 472, }"] [2025/04/06 08:25:24.824 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 473, b: 473, c: o, ahRhZ: VgauzvAfkJ, d: 08:32:43, }"] [row2="{ a: 473, b: 473, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:32:43, }"] ***************** properties ***************** "updateproportion"="0" "threadcount"="2" "operationcount"="0" "readallfields"="true" "readproportion"="0" "mysql.host"="127.0.0.1" "scanproportion"="0" "mysql.db"="multi_rocks" "table"="a3" "recordcount"="1000" "mysql.port"="4000" "workload"="core" "insertproportion"="0" "requestdistribution"="uniform" "dotransactions"="false" "mysql.user"="root" ********************************************** wait process 9377 exit for 1-th time... wait process 9377 exit for 2-th time... wait process 9377 exit for 3-th time... /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (9377) - No such process wait process 9377 exit for 4-th time... process 9377 already exit check_capture_count http://127.0.0.1:2379 0 parse error: Invalid numeric literal at line 1, column 6 run task successfully [Sun Apr 6 08:25:25 CST 2025] <<<<<< START cdc server in kill_owner_with_ddl case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kill_owner_with_ddl.95329534.out server --log-file /tmp/tidb_cdc_test/kill_owner_with_ddl/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/kill_owner_with_ddl/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2025/04/06 08:25:24.924 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 474, b: 474, c: o, ahRhZ: VgauzvAfkJ, d: 08:32:44, }"] [row2="{ b: 474, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:32:44, a: 474, }"] [2025/04/06 08:25:25.025 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ ahRhZ: VgauzvAfkJ, d: 08:32:45, a: 475, b: 475, c: o, }"] [row2="{ b: 475, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:32:45, a: 475, }"] [2025/04/06 08:25:25.125 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ ahRhZ: VgauzvAfkJ, d: 08:32:46, a: 476, b: 476, c: o, }"] [row2="{ ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:32:46, a: 476, b: 476, c: D, }"] [2025/04/06 08:25:25.225 +08:00] [WARN] [diff.go:937] ["find different row"] [column=ahRhZ] [row1="{ a: 477, b: 477, c: o, ahRhZ: VgauzvAfkJ, d: 08:32:47, }"] [row2="{ a: 477, b: 477, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:32:47, }"] [2025/04/06 08:25:25.325 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 478, b: 478, c: o, ahRhZ: VgauzvAfkJ, d: 08:32:48, }"] [row2="{ c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:32:48, a: 478, b: 478, }"] pass test: get status pass test: health [2025/04/06 08:25:24.483 +08:00] [INFO] [cases.go:432] ["check changefeed failed"] [time=0] [info=null] Run finished, takes 694.048126ms INSERT - Takes(s): 0.7, Count: 1000, OPS: 1486.2, Avg(us): 1354, Min(us): 869, Max(us): 36499, 95th(us): 2000, 99th(us): 3000 [2025/04/06 08:25:25.425 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ ahRhZ: VgauzvAfkJ, d: 08:32:49, a: 479, b: 479, c: o, }"] [row2="{ b: 479, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:32:49, a: 479, }"] [2025/04/06 08:25:25.526 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 480, b: 480, c: o, ahRhZ: VgauzvAfkJ, d: 08:41:38, }"] [row2="{ a: 480, b: 480, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:32:52, }"] [2025/04/06 08:25:25.626 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 481, b: 481, c: o, ahRhZ: VgauzvAfkJ, d: 08:32:53, }"] [row2="{ a: 481, b: 481, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:32:53, }"] [2025/04/06 08:25:25.726 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ b: 482, c: o, ahRhZ: VgauzvAfkJ, d: 08:32:54, a: 482, }"] [row2="{ d: 08:32:54, a: 482, b: 482, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, }"] [2025/04/06 08:25:25.826 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ b: 483, c: o, ahRhZ: VgauzvAfkJ, d: 08:32:55, a: 483, }"] [row2="{ a: 483, b: 483, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:32:55, }"] + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_with_auth.cli.12374.out cli unsafe reset --no-confirm --pd=http://127.0.0.1:2379 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.move_table.cli.23485.out cli tso query --pd=http://127.0.0.1:2379 ***************** properties ***************** "mysql.port"="4000" "readallfields"="true" "mysql.host"="127.0.0.1" "threadcount"="2" "scanproportion"="0" "insertproportion"="0" "mysql.user"="root" "table"="a4" "requestdistribution"="uniform" "recordcount"="1000" "readproportion"="0" "operationcount"="0" "updateproportion"="0" "dotransactions"="false" "mysql.db"="multi_rocks" "workload"="core" ********************************************** [2025/04/06 08:25:25.926 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ c: o, ahRhZ: VgauzvAfkJ, d: 08:32:56, a: 484, b: 484, }"] [row2="{ a: 484, b: 484, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:32:56, }"] [2025/04/06 08:25:26.027 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 485, b: 485, c: o, ahRhZ: VgauzvAfkJ, d: 08:32:57, }"] [row2="{ d: 08:32:57, a: 485, b: 485, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, }"] [2025/04/06 08:25:26.127 +08:00] [WARN] [diff.go:937] ["find different row"] [column=ahRhZ] [row1="{ a: 486, b: 486, c: o, ahRhZ: VgauzvAfkJ, d: 08:32:58, }"] [row2="{ a: 486, b: 486, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:32:58, }"] reset and all metadata truncated in PD! PASS [2025/04/06 08:25:26.227 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 487, b: 487, c: o, ahRhZ: VgauzvAfkJ, d: 08:32:59, }"] [row2="{ ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:32:59, a: 487, b: 487, c: D, }"] [2025/04/06 08:25:26.327 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 488, b: 488, c: o, ahRhZ: VgauzvAfkJ, d: 08:33:00, }"] [row2="{ a: 488, b: 488, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:33:00, }"] coverage: 2.0% of statements in github.com/pingcap/tiflow/... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/multi_cdc_cluster/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [2025/04/06 08:25:26.427 +08:00] [WARN] [diff.go:937] ["find different row"] [column=ahRhZ] [row1="{ d: 08:38:11, a: 489, b: 489, c: o, ahRhZ: VgauzvAfkJ, }"] [row2="{ d: 08:38:11, a: 489, b: 489, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, }"] [2025/04/06 08:25:26.528 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ c: o, ahRhZ: VgauzvAfkJ, d: 08:33:02, a: 490, b: 490, }"] [row2="{ a: 490, b: 490, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:33:02, }"] [2025/04/06 08:25:26.628 +08:00] [WARN] [diff.go:937] ["find different row"] [column=ahRhZ] [row1="{ c: o, ahRhZ: VgauzvAfkJ, d: 08:33:03, a: 491, b: 491, }"] [row2="{ b: 491, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:33:03, a: 491, }"] Run finished, takes 736.300231ms INSERT - Takes(s): 0.7, Count: 1000, OPS: 1408.1, Avg(us): 1434, Min(us): 883, Max(us): 60592, 95th(us): 2000, 99th(us): 3000 [2025/04/06 08:25:26.728 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 492, b: 492, c: o, ahRhZ: VgauzvAfkJ, d: 08:33:04, }"] [row2="{ d: 08:33:04, a: 492, b: 492, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, }"] [2025/04/06 08:25:26.828 +08:00] [WARN] [diff.go:937] ["find different row"] [column=ahRhZ] [row1="{ a: 493, b: 493, c: o, ahRhZ: VgauzvAfkJ, d: 08:33:05, }"] [row2="{ a: 493, b: 493, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:33:05, }"] [2025/04/06 08:25:26.928 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ d: 08:33:06, a: 494, b: 494, c: o, ahRhZ: VgauzvAfkJ, }"] [row2="{ a: 494, b: 494, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:33:06, }"] [2025/04/06 08:25:27.028 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 495, b: 495, c: o, ahRhZ: VgauzvAfkJ, d: 08:33:07, }"] [row2="{ c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:33:07, a: 495, b: 495, }"] [2025/04/06 08:25:27.129 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 496, b: 496, c: o, ahRhZ: VgauzvAfkJ, d: 08:33:08, }"] [row2="{ c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:33:08, a: 496, b: 496, }"] ***************** properties ***************** "insertproportion"="0" "operationcount"="0" "updateproportion"="0" "requestdistribution"="uniform" "readproportion"="0" "readallfields"="true" "threadcount"="2" "mysql.port"="4000" "mysql.db"="multi_rocks" "mysql.user"="root" "table"="a5" "recordcount"="1000" "scanproportion"="0" "mysql.host"="127.0.0.1" "dotransactions"="false" "workload"="core" ********************************************** [2025/04/06 08:25:27.033 +08:00] [INFO] [cases.go:432] ["check changefeed failed"] [time=0] [info="{\"upstream_id\":7489988795949182453,\"namespace\":\"test\",\"id\":\"changefeed-test-v2-black-hole-1\",\"sink_uri\":\"blackhole:\",\"create_time\":\"2025-04-06T08:25:24.45081282+08:00\",\"start_ts\":457152692067500036,\"config\":{\"memory_quota\":1073741824,\"case_sensitive\":false,\"force_replicate\":false,\"ignore_ineligible_table\":false,\"check_gc_safe_point\":true,\"enable_sync_point\":false,\"bdr_mode\":false,\"sync_point_interval\":600000000000,\"sync_point_retention\":86400000000000,\"filter\":{\"rules\":[\"*.*\"]},\"mounter\":{\"worker_num\":16},\"sink\":{\"csv\":{\"delimiter\":\",\",\"quote\":\"\\\"\",\"null\":\"\\\\N\",\"include_commit_ts\":false},\"transaction_atomicity\":\"\",\"encoder_concurrency\":32,\"terminator\":\"\\r\\n\",\"date_separator\":\"day\",\"enable_partition_separator\":true,\"content_compatible\":false,\"send_bootstrap_interval_in_sec\":120,\"send_bootstrap_in_msg_count\":10000,\"send_bootstrap_to_all_partition\":true,\"debezium_disable_schema\":false,\"debezium\":{\"output_old_value\":true},\"open\":{\"output_old_value\":true}},\"consistent\":{\"level\":\"none\",\"max_log_size\":64,\"flush_interval\":2000,\"meta_flush_interval\":200,\"encoding_worker_num\":16,\"flush_worker_num\":8,\"storage\":\"\",\"use_file_backend\":false},\"scheduler\":{\"enable_table_across_nodes\":false,\"region_threshold\":100000,\"write_key_threshold\":0},\"integrity\":{\"integrity_check_level\":\"none\",\"corruption_handle_level\":\"warn\"}},\"state\":\"normal\",\"creator_version\":\"v9.0.0-beta.1-9-g4c631d595\",\"resolved_ts\":0,\"checkpoint_ts\":457152692486930440,\"checkpoint_time\":\"2025-04-06 08:25:26.003\"}"] [2025/04/06 08:25:27.229 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ b: 497, c: o, ahRhZ: VgauzvAfkJ, d: 08:33:09, a: 497, }"] [row2="{ a: 497, b: 497, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:33:09, }"] [2025/04/06 08:25:27.329 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ ahRhZ: VgauzvAfkJ, d: 08:33:10, a: 498, b: 498, c: o, }"] [row2="{ a: 498, b: 498, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:33:10, }"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8302 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8302 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 06 Apr 2025 00:25:26 GMT < Content-Length: 1281 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4047b554-f313-4419-88c5-8530952b5e45 {"id":"4047b554-f313-4419-88c5-8530952b5e45","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899121} /tidb/cdc/default/__cdc_meta__/capture/9213fa17-11fa-44d3-8e58-a0c9df00462d {"id":"9213fa17-11fa-44d3-8e58-a0c9df00462d","address":"127.0.0.1:8302","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899124} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087ca1bc15 4047b554-f313-4419-88c5-8530952b5e45 /tidb/cdc/default/__cdc_meta__/owner/223196087ca1bc52 9213fa17-11fa-44d3-8e58-a0c9df00462d /tidb/cdc/default/default/upstream/7489989647983861866 {"id":7489989647983861866,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4047b554-f313-4419-88c5-8530952b5e45 {"id":"4047b554-f313-4419-88c5-8530952b5e45","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899121} /tidb/cdc/default/__cdc_meta__/capture/9213fa17-11fa-44d3-8e58-a0c9df00462d {"id":"9213fa17-11fa-44d3-8e58-a0c9df00462d","address":"127.0.0.1:8302","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899124} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087ca1bc15 4047b554-f313-4419-88c5-8530952b5e45 /tidb/cdc/default/__cdc_meta__/owner/223196087ca1bc52 9213fa17-11fa-44d3-8e58-a0c9df00462d /tidb/cdc/default/default/upstream/7489989647983861866 {"id":7489989647983861866,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4047b554-f313-4419-88c5-8530952b5e45 {"id":"4047b554-f313-4419-88c5-8530952b5e45","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899121} /tidb/cdc/default/__cdc_meta__/capture/9213fa17-11fa-44d3-8e58-a0c9df00462d {"id":"9213fa17-11fa-44d3-8e58-a0c9df00462d","address":"127.0.0.1:8302","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899124} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087ca1bc15 4047b554-f313-4419-88c5-8530952b5e45 /tidb/cdc/default/__cdc_meta__/owner/223196087ca1bc52 9213fa17-11fa-44d3-8e58-a0c9df00462d /tidb/cdc/default/default/upstream/7489989647983861866 {"id":7489989647983861866,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x check_changefeed_state http://127.0.0.1:2379 31e53092-201a-457d-a1a4-22ffeb717c55 normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=31e53092-201a-457d-a1a4-22ffeb717c55 + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c 31e53092-201a-457d-a1a4-22ffeb717c55 -s + info='{ "upstream_id": 7489989647983861866, "namespace": "default", "id": "31e53092-201a-457d-a1a4-22ffeb717c55", "state": "normal", "checkpoint_tso": 457152691012108291, "checkpoint_time": "2025-04-06 08:25:20.377", "error": null }' + echo '{ "upstream_id": 7489989647983861866, "namespace": "default", "id": "31e53092-201a-457d-a1a4-22ffeb717c55", "state": "normal", "checkpoint_tso": 457152691012108291, "checkpoint_time": "2025-04-06 08:25:20.377", "error": null }' { "upstream_id": 7489989647983861866, "namespace": "default", "id": "31e53092-201a-457d-a1a4-22ffeb717c55", "state": "normal", "checkpoint_tso": 457152691012108291, "checkpoint_time": "2025-04-06 08:25:20.377", "error": null } ++ echo '{' '"upstream_id":' 7489989647983861866, '"namespace":' '"default",' '"id":' '"31e53092-201a-457d-a1a4-22ffeb717c55",' '"state":' '"normal",' '"checkpoint_tso":' 457152691012108291, '"checkpoint_time":' '"2025-04-06' '08:25:20.377",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7489989647983861866, '"namespace":' '"default",' '"id":' '"31e53092-201a-457d-a1a4-22ffeb717c55",' '"state":' '"normal",' '"checkpoint_tso":' 457152691012108291, '"checkpoint_time":' '"2025-04-06' '08:25:20.377",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully table changefeed_auto_stop_1.usertable not exists for 1-th check, retry later [2025/04/06 08:25:27.429 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ b: 499, c: o, ahRhZ: VgauzvAfkJ, d: 08:33:11, a: 499, }"] [row2="{ b: 499, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:33:11, a: 499, }"] [2025/04/06 08:25:27.529 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 500, b: 500, c: o, ahRhZ: VgauzvAfkJ, d: 08:33:14, }"] [row2="{ c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:33:14, a: 500, b: 500, }"] [2025/04/06 08:25:27.630 +08:00] [WARN] [diff.go:937] ["find different row"] [column=ahRhZ] [row1="{ a: 501, b: 501, c: o, ahRhZ: VgauzvAfkJ, d: 08:33:15, }"] [row2="{ a: 501, b: 501, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:33:15, }"] + set +x + set +x + tso='457152692555612161 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 457152692555612161 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [2025/04/06 08:25:27.730 +08:00] [WARN] [diff.go:937] ["find different row"] [column=ahRhZ] [row1="{ a: 502, b: 502, c: o, ahRhZ: VgauzvAfkJ, d: 08:33:16, }"] [row2="{ c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:33:16, a: 502, b: 502, }"] [2025/04/06 08:25:27.830 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ c: o, ahRhZ: VgauzvAfkJ, d: 08:33:17, a: 503, b: 503, }"] [row2="{ a: 503, b: 503, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:33:17, }"] ***************** properties ***************** "mysql.user"="root" "workload"="core" "mysql.port"="4000" "dotransactions"="false" "insertproportion"="0" "threadcount"="10" "scanproportion"="0" "mysql.db"="move_table" "requestdistribution"="uniform" "readproportion"="0" "readallfields"="true" "mysql.host"="127.0.0.1" "operationcount"="0" "updateproportion"="0" "recordcount"="10000" ********************************************** [2025/04/06 08:25:27.930 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 504, b: 504, c: o, ahRhZ: VgauzvAfkJ, d: 08:33:18, }"] [row2="{ ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:33:18, a: 504, b: 504, c: D, }"] [2025/04/06 08:25:28.030 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 505, b: 505, c: o, ahRhZ: VgauzvAfkJ, d: 08:33:19, }"] [row2="{ a: 505, b: 505, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:33:19, }"] [2025/04/06 08:25:28.131 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ d: 08:33:20, a: 506, b: 506, c: o, ahRhZ: VgauzvAfkJ, }"] [row2="{ d: 08:33:20, a: 506, b: 506, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, }"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 06 Apr 2025 00:25:28 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/9d7077c4-d482-42fd-b85e-bd9b18f0d9ec {UpstreamID:7489989421666590001 Namespace:default ID:9d7077c4-d482-42fd-b85e-bd9b18f0d9ec SinkURI:mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1 CreateTime:2025-04-06 08:24:31.34006381 +0800 CST StartTs:457152678123012101 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0043c6cf0 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457152678149226502} {CheckpointTs:457152680141520919 MinTableBarrierTs:457152680154628120 AdminJobType:noop} span: {tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}, resolvedTs: 457152680141520919, checkpointTs: 457152680141520919, state: Preparing span: {tableID:116, startKey:7480000000000000ff745f720000000000fa, endKey:7480000000000000ff745f730000000000fa}, resolvedTs: 457152680141520919, checkpointTs: 457152680141520919, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c6337b01-1464-4b10-a554-413b8b0aeee2 {"id":"c6337b01-1464-4b10-a554-413b8b0aeee2","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899125} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087bdb3a92 c6337b01-1464-4b10-a554-413b8b0aeee2 /tidb/cdc/default/default/changefeed/info/9d7077c4-d482-42fd-b85e-bd9b18f0d9ec {"upstream-id":7489989421666590001,"namespace":"default","changefeed-id":"9d7077c4-d482-42fd-b85e-bd9b18f0d9ec","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2025-04-06T08:24:31.34006381+08:00","start-ts":457152678123012101,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457152678149226502} /tidb/cdc/default/default/changefeed/status/9d7077c4-d482-42fd-b85e-bd9b18f0d9ec {"checkpoint-ts":457152680141520919,"min-table-barrier-ts":457152680154628120,"admin-job-type":0} /tidb/cdc/default/default/task/position/c6337b01-1464-4b10-a554-413b8b0aeee2/9d7077c4-d482-42fd-b85e-bd9b18f0d9ec {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489989421666590001 {"id":7489989421666590001,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/9d7077c4-d482-42fd-b85e-bd9b18f0d9ec {UpstreamID:7489989421666590001 Namespace:default ID:9d7077c4-d482-42fd-b85e-bd9b18f0d9ec SinkURI:mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1 CreateTime:2025-04-06 08:24:31.34006381 +0800 CST StartTs:457152678123012101 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0043c6cf0 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457152678149226502} {CheckpointTs:457152680141520919 MinTableBarrierTs:457152680154628120 AdminJobType:noop} span: {tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}, resolvedTs: 457152680141520919, checkpointTs: 457152680141520919, state: Preparing span: {tableID:116, startKey:7480000000000000ff745f720000000000fa, endKey:7480000000000000ff745f730000000000fa}, resolvedTs: 457152680141520919, checkpointTs: 457152680141520919, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c6337b01-1464-4b10-a554-413b8b0aeee2 {"id":"c6337b01-1464-4b10-a554-413b8b0aeee2","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899125} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087bdb3a92 c6337b01-1464-4b10-a554-413b8b0aeee2 /tidb/cdc/default/default/changefeed/info/9d7077c4-d482-42fd-b85e-bd9b18f0d9ec {"upstream-id":7489989421666590001,"namespace":"default","changefeed-id":"9d7077c4-d482-42fd-b85e-bd9b18f0d9ec","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2025-04-06T08:24:31.34006381+08:00","start-ts":457152678123012101,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457152678149226502} /tidb/cdc/default/default/changefeed/status/9d7077c4-d482-42fd-b85e-bd9b18f0d9ec {"checkpoint-ts":457152680141520919,"min-table-barrier-ts":457152680154628120,"admin-job-type":0} /tidb/cdc/default/default/task/position/c6337b01-1464-4b10-a554-413b8b0aeee2/9d7077c4-d482-42fd-b85e-bd9b18f0d9ec {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489989421666590001 {"id":7489989421666590001,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/9d7077c4-d482-42fd-b85e-bd9b18f0d9ec {UpstreamID:7489989421666590001 Namespace:default ID:9d7077c4-d482-42fd-b85e-bd9b18f0d9ec SinkURI:mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1 CreateTime:2025-04-06 08:24:31.34006381 +0800 CST StartTs:457152678123012101 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0043c6cf0 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457152678149226502} {CheckpointTs:457152680141520919 MinTableBarrierTs:457152680154628120 AdminJobType:noop} span: {tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}, resolvedTs: 457152680141520919, checkpointTs: 457152680141520919, state: Preparing span: {tableID:116, startKey:7480000000000000ff745f720000000000fa, endKey:7480000000000000ff745f730000000000fa}, resolvedTs: 457152680141520919, checkpointTs: 457152680141520919, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c6337b01-1464-4b10-a554-413b8b0aeee2 {"id":"c6337b01-1464-4b10-a554-413b8b0aeee2","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899125} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087bdb3a92 c6337b01-1464-4b10-a554-413b8b0aeee2 /tidb/cdc/default/default/changefeed/info/9d7077c4-d482-42fd-b85e-bd9b18f0d9ec {"upstream-id":7489989421666590001,"namespace":"default","changefeed-id":"9d7077c4-d482-42fd-b85e-bd9b18f0d9ec","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2025-04-06T08:24:31.34006381+08:00","start-ts":457152678123012101,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457152678149226502} /tidb/cdc/default/default/changefeed/status/9d7077c4-d482-42fd-b85e-bd9b18f0d9ec {"checkpoint-ts":457152680141520919,"min-table-barrier-ts":457152680154628120,"admin-job-type":0} /tidb/cdc/default/default/task/position/c6337b01-1464-4b10-a554-413b8b0aeee2/9d7077c4-d482-42fd-b85e-bd9b18f0d9ec {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489989421666590001 {"id":7489989421666590001,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x check_capture_count http://127.0.0.1:2379 1 run task successfully table kill_owner_with_ddl.t1 exists table kill_owner_with_ddl.t2 exists table kill_owner_with_ddl.t3 not exists for 1-th check, retry later [2025/04/06 08:25:28.231 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 507, b: 507, c: o, ahRhZ: VgauzvAfkJ, d: 08:33:21, }"] [row2="{ a: 507, b: 507, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:33:21, }"] [2025/04/06 08:25:28.331 +08:00] [WARN] [diff.go:937] ["find different row"] [column=ahRhZ] [row1="{ c: o, ahRhZ: VgauzvAfkJ, d: 08:33:22, a: 508, b: 508, }"] [row2="{ c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:33:22, a: 508, b: 508, }"] table ddl_attributes.attributes_t1_new exists table ddl_attributes.finish_mark not exists for 1-th check, retry later nonempty select id, val from test.availability1 where id=3 and val=3 run task successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... Run finished, takes 1.119895993s INSERT - Takes(s): 1.1, Count: 1000, OPS: 909.1, Avg(us): 2201, Min(us): 898, Max(us): 75050, 95th(us): 3000, 99th(us): 48000 [2025/04/06 08:25:28.431 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ ahRhZ: VgauzvAfkJ, d: 08:33:23, a: 509, b: 509, c: o, }"] [row2="{ a: 509, b: 509, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:33:23, }"] [2025/04/06 08:25:28.532 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ ahRhZ: VgauzvAfkJ, d: 08:33:24, a: 510, b: 510, c: o, }"] [row2="{ a: 510, b: 510, c: N, ahRhZ: ZVl9wSo1giq0Z, d: 08:33:24, }"] [2025/04/06 08:25:28.632 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 511, b: 511, c: o, ahRhZ: VgauzvAfkJ, d: 08:33:25, }"] [row2="{ a: 511, b: 511, c: fWBA, ahRhZ: sZPmSx1RIWG, d: 08:33:25, }"] table ddl_attributes.finish_mark not exists for 2-th check, retry later [2025/04/06 08:25:28.732 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ d: 08:33:26, a: 512, b: 512, c: o, ahRhZ: VgauzvAfkJ, }"] [row2="{ d: 08:33:26, a: 512, b: 512, c: T0ou, ahRhZ: jpimSpKXM0IZRLcM5iv7MZ2vN2RF8UPIt0z, }"] [2025/04/06 08:25:28.832 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 513, b: 513, c: o, ahRhZ: VgauzvAfkJ, d: 08:33:27, }"] [row2="{ a: 513, b: 513, c: j, ahRhZ: Kk, d: 08:33:27, }"] wait process cdc.test exit for 3-th time... [2025/04/06 08:25:28.932 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ c: o, ahRhZ: VgauzvAfkJ, d: 08:33:28, a: 514, b: 514, }"] [row2="{ a: 514, b: 514, c: 2Ed8RxdJ, ahRhZ: LJpRMs0CYwIFMN6AH4UaaTQ9i, d: 08:33:28, }"] [2025/04/06 08:25:29.033 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 515, b: 515, c: o, ahRhZ: VgauzvAfkJ, d: 08:33:29, }"] [row2="{ a: 515, b: 515, c: cq2N, ahRhZ: swEKUtrrMA1bIV4HoRkPipq9, d: 08:33:29, }"] [2025/04/06 08:25:29.133 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ d: 08:33:30, a: 516, b: 516, c: o, ahRhZ: VgauzvAfkJ, }"] [row2="{ a: 516, b: 516, c: 6F4bCPBZtd, ahRhZ: pSuSWtWPdLYfsTGleCHuWylG4qD, d: 08:33:30, }"] [2025/04/06 08:25:29.233 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ d: 08:33:31, a: 517, b: 517, c: o, ahRhZ: VgauzvAfkJ, }"] [row2="{ ahRhZ: WEgrtv92Z3FX9PGX3GKNrWP7jCuWEBRsjQ9dx8YA, d: 08:33:31, a: 517, b: 517, c: ryAxr7Lnl, }"] [2025/04/06 08:25:29.333 +08:00] [WARN] [diff.go:937] ["find different row"] [column=ahRhZ] [row1="{ a: 518, b: 518, c: o, ahRhZ: VgauzvAfkJ, d: 08:33:32, }"] [row2="{ ahRhZ: 7ZgFRK17GgjXyMNwSnyMbC0lf3n2u1WuAmPMn1, d: 08:33:32, a: 518, b: 518, c: uCmUYzIK, }"] cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit run test case test_expire_capture [Sun Apr 6 08:25:29 CST 2025] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.64776479.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_expire_capture.server1.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_expire_capture.server1 --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2025/04/06 08:25:29.433 +08:00] [WARN] [diff.go:937] ["find different row"] [column=ahRhZ] [row1="{ a: 519, b: 519, c: o, ahRhZ: VgauzvAfkJ, d: 08:33:33, }"] [row2="{ d: 08:33:33, a: 519, b: 519, c: Vv4mYJwjIG, ahRhZ: lZ7TBN, }"] [2025/04/06 08:25:29.534 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ d: 08:33:44, a: 530, b: 530, c: o, ahRhZ: VgauzvAfkJ, }"] [2025/04/06 08:25:29.634 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 531, b: 531, c: o, ahRhZ: VgauzvAfkJ, d: 08:33:45, }"] [2025/04/06 08:25:29.734 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ ahRhZ: VgauzvAfkJ, d: 08:33:46, a: 532, b: 532, c: o, }"] [2025/04/06 08:25:29.834 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 533, b: 533, c: o, ahRhZ: VgauzvAfkJ, d: 08:33:47, }"] [2025/04/06 08:25:29.935 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 534, b: 534, c: o, ahRhZ: VgauzvAfkJ, d: 08:33:48, }"] table multi_rocks.finish_mark not exists for 1-th check, retry later table changefeed_auto_stop_1.usertable exists table changefeed_auto_stop_2.usertable not exists for 1-th check, retry later [2025/04/06 08:25:30.035 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 535, b: 535, c: o, ahRhZ: VgauzvAfkJ, d: 08:33:49, }"] [2025/04/06 08:25:30.135 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 536, b: 536, c: o, ahRhZ: VgauzvAfkJ, d: 08:33:50, }"] [2025/04/06 08:25:30.235 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ d: 08:33:51, a: 537, b: 537, c: o, ahRhZ: VgauzvAfkJ, }"] [2025/04/06 08:25:30.335 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ b: 538, c: o, ahRhZ: VgauzvAfkJ, d: 08:33:52, a: 538, }"] [2025/04/06 08:25:30.435 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 539, b: 539, c: o, ahRhZ: VgauzvAfkJ, d: 08:33:53, }"] table ddl_attributes.finish_mark not exists for 3-th check, retry later [2025/04/06 08:25:30.536 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ ahRhZ: VgauzvAfkJ, d: 08:33:54, a: 540, b: 540, c: o, }"] [2025/04/06 08:25:30.636 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 541, b: 541, c: o, ahRhZ: VgauzvAfkJ, d: 08:33:55, }"] start tidb cluster in /tmp/tidb_cdc_test/multi_cdc_cluster Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... table kill_owner_with_ddl.t3 exists check diff failed 1-th time, retry later [2025/04/06 08:25:30.736 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 542, b: 542, c: o, ahRhZ: VgauzvAfkJ, d: 08:33:56, }"] [2025/04/06 08:25:30.836 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ c: o, ahRhZ: VgauzvAfkJ, d: 08:33:57, a: 543, b: 543, }"] [2025/04/06 08:25:30.936 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 544, b: 544, c: o, ahRhZ: VgauzvAfkJ, d: 08:33:58, }"] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_with_auth.cli.12471.out cli unsafe resolve-lock --region=96 [2025/04/06 08:25:31.037 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ ahRhZ: VgauzvAfkJ, d: 08:33:59, a: 545, b: 545, c: o, }"] [2025/04/06 08:25:31.137 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ d: 08:34:00, a: 546, b: 546, c: o, ahRhZ: VgauzvAfkJ, }"] PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... Run finished, takes 2.8924166s INSERT - Takes(s): 2.9, Count: 10000, OPS: 3464.6, Avg(us): 2838, Min(us): 982, Max(us): 315524, 95th(us): 3000, 99th(us): 37000 [Sun Apr 6 08:25:30 CST 2025] <<<<<< START cdc server in move_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.move_table.2357723579.out server --log-file /tmp/tidb_cdc_test/move_table/cdc1.log --log-level debug --data-dir /tmp/tidb_cdc_test/move_table/cdc_data1 --cluster-id default --addr 127.0.0.1:8300 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2025/04/06 08:25:31.237 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ c: o, ahRhZ: VgauzvAfkJ, d: 08:34:01, a: 547, b: 547, }"] [2025/04/06 08:25:31.337 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ d: 08:34:02, a: 548, b: 548, c: o, ahRhZ: VgauzvAfkJ, }"] [2025/04/06 08:25:31.438 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ d: 08:34:03, a: 549, b: 549, c: o, ahRhZ: VgauzvAfkJ, }"] [2025/04/06 08:25:31.538 +08:00] [WARN] [diff.go:937] ["find different row"] [column=ahRhZ] [row1="{ a: 550, b: 550, c: o, ahRhZ: VgauzvAfkJ, d: 08:34:04, }"] [row2="{ ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:34:04, a: 550, b: 550, c: dXRs4mkPlK, }"] [2025/04/06 08:25:31.638 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ d: 08:34:05, a: 551, b: 551, c: o, ahRhZ: VgauzvAfkJ, }"] [row2="{ d: 08:34:05, a: 551, b: 551, c: fW9b5gV, ahRhZ: QdArKAP2RgOEVYU7Y, }"] table changefeed_auto_stop_2.usertable not exists for 2-th check, retry later [2025/04/06 08:25:31.738 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ b: 552, c: o, ahRhZ: VgauzvAfkJ, d: 08:34:06, a: 552, }"] [row2="{ a: 552, b: 552, c: XCBrTwdc9, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:34:06, }"] [2025/04/06 08:25:31.839 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 553, b: 553, c: o, ahRhZ: VgauzvAfkJ, d: 08:34:07, }"] [row2="{ a: 553, b: 553, c: s7docm6h, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:34:07, }"] [2025/04/06 08:25:31.939 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 554, b: 554, c: o, ahRhZ: VgauzvAfkJ, d: 08:34:08, }"] [row2="{ c: A, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:34:08, a: 554, b: 554, }"] [2025/04/06 08:25:32.039 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ d: 08:34:09, a: 555, b: 555, c: o, ahRhZ: VgauzvAfkJ, }"] [row2="{ a: 555, b: 555, c: EPsxp, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:34:09, }"] [2025/04/06 08:25:32.139 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 556, b: 556, c: o, ahRhZ: VgauzvAfkJ, d: 08:34:10, }"] [row2="{ c: boRHT, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:34:10, a: 556, b: 556, }"] table multi_rocks.finish_mark not exists for 2-th check, retry later [2025/04/06 08:25:32.240 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ ahRhZ: VgauzvAfkJ, d: 08:34:11, a: 557, b: 557, c: o, }"] [row2="{ a: 557, b: 557, c: crXXW, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:34:11, }"] [2025/04/06 08:25:32.340 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ d: 08:34:12, a: 558, b: 558, c: o, ahRhZ: VgauzvAfkJ, }"] [row2="{ ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:34:12, a: 558, b: 558, c: S02BGn, }"] [2025/04/06 08:25:32.440 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 559, b: 559, c: o, ahRhZ: VgauzvAfkJ, d: 08:34:13, }"] [row2="{ b: 559, c: nOLmDW2QYo, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:34:13, a: 559, }"] [2025/04/06 08:25:31.783 +08:00] [INFO] [cases.go:432] ["check changefeed failed"] [time=0] [info="{\"upstream_id\":7489988795949182453,\"namespace\":\"test\",\"id\":\"changefeed-test-v2-black-hole-1\",\"sink_uri\":\"blackhole:\",\"create_time\":\"2025-04-06T08:25:24.45081282+08:00\",\"start_ts\":457152692067500036,\"admin_job_type\":1,\"config\":{\"memory_quota\":1123450,\"case_sensitive\":false,\"force_replicate\":false,\"ignore_ineligible_table\":false,\"check_gc_safe_point\":false,\"enable_sync_point\":false,\"bdr_mode\":false,\"sync_point_interval\":600000000000,\"sync_point_retention\":86400000000000,\"filter\":{\"rules\":[\"a.d\",\"b.x\"],\"ignore_txn_start_ts\":[1,2,3],\"event_filters\":[{\"matcher\":[\"test.worker\"],\"ignore_event\":[\"update\"],\"ignore_sql\":[\"^drop\",\"add column\"],\"ignore_insert_value_expr\":\"id \\u003e= 100\",\"ignore_update_new_value_expr\":\"gender = 'male'\",\"ignore_update_old_value_expr\":\"age \\u003c 18\",\"ignore_delete_value_expr\":\"id \\u003e 100\"}]},\"mounter\":{\"worker_num\":17},\"sink\":{\"protocol\":\"arvo\",\"csv\":{\"delimiter\":\",\",\"quote\":\"\\\"\",\"null\":\"\\\\N\",\"include_commit_ts\":false},\"column_selectors\":[{\"matcher\":[\"a.b\"],\"columns\":[\"c\"]}],\"transaction_atomicity\":\"table\",\"encoder_concurrency\":32,\"terminator\":\"a\",\"date_separator\":\"day\",\"enable_partition_separator\":true,\"content_compatible\":true,\"send_bootstrap_interval_in_sec\":120,\"send_bootstrap_in_msg_count\":10000,\"send_bootstrap_to_all_partition\":true,\"debezium_disable_schema\":true,\"debezium\":{\"output_old_value\":true},\"open\":{\"output_old_value\":true}},\"consistent\":{\"level\":\"none\",\"max_log_size\":64,\"flush_interval\":2000,\"meta_flush_interval\":200,\"encoding_worker_num\":31,\"flush_worker_num\":18,\"storage\":\"\",\"use_file_backend\":false},\"scheduler\":{\"enable_table_across_nodes\":false,\"region_threshold\":13,\"write_key_threshold\":0},\"integrity\":{\"integrity_check_level\":\"none\",\"corruption_handle_level\":\"warn\"}},\"state\":\"stopped\",\"creator_version\":\"v9.0.0-beta.1-9-g4c631d595\",\"resolved_ts\":457152692486930440,\"checkpoint_ts\":457152692486930440,\"checkpoint_time\":\"2025-04-06 08:25:26.003\"}"] + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_with_auth.cli.12495.out cli unsafe resolve-lock --region=96 --ts=457152692942274564 check diff successfully [2025/04/06 08:25:32.540 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 560, b: 560, c: o, ahRhZ: VgauzvAfkJ, d: 08:34:14, }"] [2025/04/06 08:25:32.641 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 561, b: 561, c: o, ahRhZ: VgauzvAfkJ, d: 08:34:15, }"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 06 Apr 2025 00:25:32 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/e684662d-022f-4a79-9ef3-0a59dba96193 {UpstreamID:7489988628380945915 Namespace:default ID:e684662d-022f-4a79-9ef3-0a59dba96193 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-06 08:21:28.032207553 +0800 CST StartTs:457152628763131907 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003f333b0 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457152630100066306} {CheckpointTs:457152692634255363 MinTableBarrierTs:457152693944975364 AdminJobType:noop} span: {tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}, resolvedTs: 457152692634255363, checkpointTs: 457152692634255363, state: Preparing span: {tableID:112, startKey:7480000000000000ff705f720000000000fa, endKey:7480000000000000ff705f730000000000fa}, resolvedTs: 457152692634255363, checkpointTs: 457152692634255363, state: Preparing span: {tableID:116, startKey:7480000000000000ff745f720000000000fa, endKey:7480000000000000ff745f730000000000fa}, resolvedTs: 457152692634255363, checkpointTs: 457152692634255363, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f5addd6e-0da2-4e85-bb44-d57c29919b1f {"id":"f5addd6e-0da2-4e85-bb44-d57c29919b1f","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899129} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 4 /tidb/cdc/default/__cdc_meta__/owner/2231960879055079 f5addd6e-0da2-4e85-bb44-d57c29919b1f /tidb/cdc/default/default/changefeed/info/e684662d-022f-4a79-9ef3-0a59dba96193 {"upstream-id":7489988628380945915,"namespace":"default","changefeed-id":"e684662d-022f-4a79-9ef3-0a59dba96193","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-06T08:21:28.032207553+08:00","start-ts":457152628763131907,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457152630100066306} /tidb/cdc/default/default/changefeed/status/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":457152692634255363,"min-table-barrier-ts":457152693944975364,"admin-job-type":0} /tidb/cdc/default/default/task/position/f5addd6e-0da2-4e85-bb44-d57c29919b1f/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489988628380945915 {"id":7489988628380945915,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/e684662d-022f-4a79-9ef3-0a59dba96193 {UpstreamID:7489988628380945915 Namespace:default ID:e684662d-022f-4a79-9ef3-0a59dba96193 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-06 08:21:28.032207553 +0800 CST StartTs:457152628763131907 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003f333b0 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457152630100066306} {CheckpointTs:457152692634255363 MinTableBarrierTs:457152693944975364 AdminJobType:noop} span: {tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}, resolvedTs: 457152692634255363, checkpointTs: 457152692634255363, state: Preparing span: {tableID:112, startKey:7480000000000000ff705f720000000000fa, endKey:7480000000000000ff705f730000000000fa}, resolvedTs: 457152692634255363, checkpointTs: 457152692634255363, state: Preparing span: {tableID:116, startKey:7480000000000000ff745f720000000000fa, endKey:7480000000000000ff745f730000000000fa}, resolvedTs: 457152692634255363, checkpointTs: 457152692634255363, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f5addd6e-0da2-4e85-bb44-d57c29919b1f {"id":"f5addd6e-0da2-4e85-bb44-d57c29919b1f","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899129} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 4 /tidb/cdc/default/__cdc_meta__/owner/2231960879055079 f5addd6e-0da2-4e85-bb44-d57c29919b1f /tidb/cdc/default/default/changefeed/info/e684662d-022f-4a79-9ef3-0a59dba96193 {"upstream-id":7489988628380945915,"namespace":"default","changefeed-id":"e684662d-022f-4a79-9ef3-0a59dba96193","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-06T08:21:28.032207553+08:00","start-ts":457152628763131907,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457152630100066306} /tidb/cdc/default/default/changefeed/status/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":457152692634255363,"min-table-barrier-ts":457152693944975364,"admin-job-type":0} /tidb/cdc/default/default/task/position/f5addd6e-0da2-4e85-bb44-d57c29919b1f/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489988628380945915 {"id":7489988628380945915,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/e684662d-022f-4a79-9ef3-0a59dba96193 {UpstreamID:7489988628380945915 Namespace:default ID:e684662d-022f-4a79-9ef3-0a59dba96193 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-06 08:21:28.032207553 +0800 CST StartTs:457152628763131907 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003f333b0 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457152630100066306} {CheckpointTs:457152692634255363 MinTableBarrierTs:457152693944975364 AdminJobType:noop} span: {tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}, resolvedTs: 457152692634255363, checkpointTs: 457152692634255363, state: Preparing span: {tableID:112, startKey:7480000000000000ff705f720000000000fa, endKey:7480000000000000ff705f730000000000fa}, resolvedTs: 457152692634255363, checkpointTs: 457152692634255363, state: Preparing span: {tableID:116, startKey:7480000000000000ff745f720000000000fa, endKey:7480000000000000ff745f730000000000fa}, resolvedTs: 457152692634255363, checkpointTs: 457152692634255363, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f5addd6e-0da2-4e85-bb44-d57c29919b1f {"id":"f5addd6e-0da2-4e85-bb44-d57c29919b1f","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899129} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 4 /tidb/cdc/default/__cdc_meta__/owner/2231960879055079 f5addd6e-0da2-4e85-bb44-d57c29919b1f /tidb/cdc/default/default/changefeed/info/e684662d-022f-4a79-9ef3-0a59dba96193 {"upstream-id":7489988628380945915,"namespace":"default","changefeed-id":"e684662d-022f-4a79-9ef3-0a59dba96193","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-06T08:21:28.032207553+08:00","start-ts":457152628763131907,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457152630100066306} /tidb/cdc/default/default/changefeed/status/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":457152692634255363,"min-table-barrier-ts":457152693944975364,"admin-job-type":0} /tidb/cdc/default/default/task/position/f5addd6e-0da2-4e85-bb44-d57c29919b1f/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489988628380945915 {"id":7489988628380945915,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x cdc.test cli capture list 2>&1 | grep '"is-owner": true' [2025/04/06 08:25:32.741 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 562, b: 562, c: o, ahRhZ: VgauzvAfkJ, d: 08:34:16, }"] [2025/04/06 08:25:32.841 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ d: 08:34:17, a: 563, b: 563, c: o, ahRhZ: VgauzvAfkJ, }"] [2025/04/06 08:25:32.941 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 564, b: 564, c: o, ahRhZ: VgauzvAfkJ, d: 08:34:18, }"] start tidb cluster in /tmp/tidb_cdc_test/sql_mode Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... table ddl_attributes.finish_mark not exists for 4-th check, retry later PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... wait process cdc.test exit for 1-th time... [2025/04/06 08:25:33.041 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 565, b: 565, c: o, ahRhZ: VgauzvAfkJ, d: 08:34:19, }"] [2025/04/06 08:25:33.142 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ d: 08:34:20, a: 566, b: 566, c: o, ahRhZ: VgauzvAfkJ, }"] Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release [2025/04/06 08:25:33.242 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ d: 08:34:21, a: 567, b: 567, c: o, ahRhZ: VgauzvAfkJ, }"] [2025/04/06 08:25:33.343 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ d: 08:34:22, a: 568, b: 568, c: o, ahRhZ: VgauzvAfkJ, }"] [2025/04/06 08:25:33.443 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ c: o, ahRhZ: VgauzvAfkJ, d: 08:34:23, a: 569, b: 569, }"] [2025/04/06 08:25:33.543 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 570, b: 570, c: o, ahRhZ: VgauzvAfkJ, d: 08:34:24, }"] [2025/04/06 08:25:33.643 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ d: 08:34:25, a: 571, b: 571, c: o, ahRhZ: VgauzvAfkJ, }"] [2025/04/06 08:25:33.743 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 572, b: 572, c: o, ahRhZ: VgauzvAfkJ, d: 08:34:26, }"] table changefeed_auto_stop_2.usertable exists table changefeed_auto_stop_3.usertable not exists for 1-th check, retry later wait process cdc.test exit for 2-th time... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > [2025/04/06 08:25:33.844 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ c: o, ahRhZ: VgauzvAfkJ, d: 08:34:27, a: 573, b: 573, }"] [2025/04/06 08:25:33.944 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ c: o, ahRhZ: VgauzvAfkJ, d: 08:34:28, a: 574, b: 574, }"] < HTTP/1.1 200 OK < Date: Sun, 06 Apr 2025 00:25:33 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6f229d06-016c-4169-bab4-9209ad4184b8 {"id":"6f229d06-016c-4169-bab4-9209ad4184b8","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899131} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087cb9b505 6f229d06-016c-4169-bab4-9209ad4184b8 /tidb/cdc/default/default/upstream/7489989674838134457 {"id":7489989674838134457,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6f229d06-016c-4169-bab4-9209ad4184b8 {"id":"6f229d06-016c-4169-bab4-9209ad4184b8","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899131} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087cb9b505 6f229d06-016c-4169-bab4-9209ad4184b8 /tidb/cdc/default/default/upstream/7489989674838134457 {"id":7489989674838134457,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6f229d06-016c-4169-bab4-9209ad4184b8 {"id":"6f229d06-016c-4169-bab4-9209ad4184b8","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899131} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087cb9b505 6f229d06-016c-4169-bab4-9209ad4184b8 /tidb/cdc/default/default/upstream/7489989674838134457 {"id":7489989674838134457,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.move_table.cli.23631.out cli changefeed create --start-ts=457152692555612161 '--sink-uri=mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1' "is-owner": true, run task successfully [2025/04/06 08:25:34.044 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ b: 575, c: o, ahRhZ: VgauzvAfkJ, d: 08:34:29, a: 575, }"] [2025/04/06 08:25:34.144 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 576, b: 576, c: o, ahRhZ: VgauzvAfkJ, d: 08:34:30, }"] [2025/04/06 08:25:34.244 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 577, b: 577, c: o, ahRhZ: VgauzvAfkJ, d: 08:34:31, }"] table multi_rocks.finish_mark not exists for 3-th check, retry later wait process cdc.test exit for 3-th time... pass test: changefeed apis [2025/04/06 08:25:34.382 +08:00] [INFO] [cases.go:432] ["check changefeed failed"] [time=0] [info=null] [2025/04/06 08:25:34.345 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ ahRhZ: VgauzvAfkJ, d: 08:34:32, a: 578, b: 578, c: o, }"] [2025/04/06 08:25:34.445 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 579, b: 579, c: o, ahRhZ: VgauzvAfkJ, d: 08:34:33, }"] + set +x % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 218 100 212 100 6 156k 4535 --:--:-- --:--:-- --:--:-- 207k { "error_msg": "[CDC:ErrAPIInvalidParam]invalid log level: json: cannot unmarshal string into Go value of type struct { Level string \"json:\\\"log_level\\\"\" }", "error_code": "CDC:ErrAPIInvalidParam" [2025/04/06 08:25:34.545 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 580, b: 580, c: aegrrFw, ahRhZ: VgauzvAfkJ, d: 08:34:34, }"] [2025/04/06 08:25:34.645 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 581, b: 581, c: wB9s4jP, ahRhZ: VgauzvAfkJ, d: 08:34:35, }"] [2025/04/06 08:25:34.745 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ b: 582, c: C0mr4eI, ahRhZ: VgauzvAfkJ, d: 08:34:36, a: 582, }"] cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Sun Apr 6 08:25:34 CST 2025] <<<<<< run test case kill_owner_with_ddl success! >>>>>> table ddl_attributes.finish_mark not exists for 5-th check, retry later [2025/04/06 08:25:34.845 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 583, b: 583, c: VRRdNw, ahRhZ: VgauzvAfkJ, d: 08:34:37, }"] [2025/04/06 08:25:34.946 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 584, b: 584, c: bhXCSg, ahRhZ: VgauzvAfkJ, d: 08:34:38, }"] Starting Upstream TiDB... [2025/04/06 08:25:35.046 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ b: 585, c: YwSKKJfDdB, ahRhZ: VgauzvAfkJ, d: 08:34:39, a: 585, }"] [2025/04/06 08:25:35.146 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 586, b: 586, c: VkyKcntF, ahRhZ: VgauzvAfkJ, d: 08:34:40, }"] [2025/04/06 08:25:35.246 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 587, b: 587, c: AIqgvelY, ahRhZ: VgauzvAfkJ, d: 08:34:41, }"] Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Create changefeed successfully! ID: 44202c40-4a17-4681-8452-63490f9b5e23 Info: {"upstream_id":7489989674838134457,"namespace":"default","id":"44202c40-4a17-4681-8452-63490f9b5e23","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create_time":"2025-04-06T08:25:34.931675019+08:00","start_ts":457152692555612161,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457152692555612161,"checkpoint_ts":457152692555612161,"checkpoint_time":"2025-04-06 08:25:26.265"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... [2025/04/06 08:25:35.346 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ c: 0izypbZqT, ahRhZ: VgauzvAfkJ, d: 08:34:42, a: 588, b: 588, }"] [2025/04/06 08:25:35.447 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ b: 589, c: iIC5x, ahRhZ: VgauzvAfkJ, d: 08:34:43, a: 589, }"] owner pid: 6482 owner id f5addd6e-0da2-4e85-bb44-d57c29919b1f process status: T ETCDCTL_API=3 etcdctl get /tidb/cdc/default/__cdc_meta__/owner --prefix | grep -v 'f5addd6e-0da2-4e85-bb44-d57c29919b1f' table changefeed_auto_stop_3.usertable exists table changefeed_auto_stop_4.usertable not exists for 1-th check, retry later [2025/04/06 08:25:35.547 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ b: 590, c: o, ahRhZ: VgauzvAfkJ, d: 08:34:44, a: 590, }"] [row2="{ a: 590, b: 590, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:34:44, }"] [2025/04/06 08:25:35.647 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ d: 08:34:45, a: 591, b: 591, c: o, ahRhZ: VgauzvAfkJ, }"] [row2="{ a: 591, b: 591, c: D, ahRhZ: 6mz7lAogb4fbFUTFwh7KiN, d: 08:34:45, }"] [2025/04/06 08:25:35.747 +08:00] [WARN] [diff.go:937] ["find different row"] [column=ahRhZ] [row1="{ a: 592, b: 592, c: o, ahRhZ: VgauzvAfkJ, d: 08:34:46, }"] [row2="{ a: 592, b: 592, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:34:46, }"] /tidb/cdc/default/__cdc_meta__/owner/2231960879055079 run task successfully process status: S nonempty select id, val from test.availability1 where id=2 and val=22 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later [2025/04/06 08:25:35.847 +08:00] [WARN] [diff.go:937] ["find different row"] [column=ahRhZ] [row1="{ b: 593, c: o, ahRhZ: VgauzvAfkJ, d: 08:34:47, a: 593, }"] [row2="{ d: 08:34:47, a: 593, b: 593, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, }"] [2025/04/06 08:25:35.947 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ d: 08:34:48, a: 594, b: 594, c: o, ahRhZ: VgauzvAfkJ, }"] [row2="{ a: 594, b: 594, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:34:48, }"] table multi_rocks.finish_mark exists check diff successfully Verifying downstream PD is started... [2025/04/06 08:25:36.048 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 595, b: 595, c: o, ahRhZ: VgauzvAfkJ, d: 08:34:49, }"] [row2="{ d: 08:34:49, a: 595, b: 595, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, }"] [2025/04/06 08:25:36.148 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ c: o, ahRhZ: VgauzvAfkJ, d: 08:34:50, a: 596, b: 596, }"] [row2="{ a: 596, b: 596, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:34:50, }"] [2025/04/06 08:25:36.248 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ b: 597, c: o, ahRhZ: VgauzvAfkJ, d: 08:34:51, a: 597, }"] [row2="{ a: 597, b: 597, c: D, ahRhZ: MoJZFXkxBYsFSY6SO2CeORXVMeyGx9eMPGcW5mO, d: 08:34:51, }"] [2025/04/06 08:25:36.348 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 598, b: 598, c: o, ahRhZ: VgauzvAfkJ, d: 08:34:52, }"] [row2="{ a: 598, b: 598, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:34:52, }"] [2025/04/06 08:25:36.448 +08:00] [WARN] [diff.go:937] ["find different row"] [column=ahRhZ] [row1="{ a: 599, b: 599, c: o, ahRhZ: VgauzvAfkJ, d: 08:34:53, }"] [row2="{ a: 599, b: 599, c: D, ahRhZ: QdArKAP2RgOEVYU7Y, d: 08:34:53, }"] wait process cdc.test exit for 1-th time... [2025/04/06 08:25:36.549 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 610, b: 610, c: Ppoa2, ahRhZ: S8OClp9UE, d: 08:35:06, }"] [2025/04/06 08:25:36.649 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ d: 08:35:07, a: 611, b: 611, c: efLNAhdiPb, ahRhZ: gyyGc7m11sY1my53n2PrX4bjAEHKhDD, }"] [2025/04/06 08:25:36.749 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ d: 08:35:08, a: 612, b: 612, c: AndB, ahRhZ: rXCUlV3Mt1yXNuqG8yH, }"] table ddl_attributes.finish_mark not exists for 6-th check, retry later [2025/04/06 08:25:36.849 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 613, b: 613, c: rQWZQ, ahRhZ: KIPOb294lucePkF7an, d: 08:35:09, }"] [2025/04/06 08:25:36.950 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 614, b: 614, c: LW6o9gxgj, ahRhZ: akHFyT1Yhn3Ftbq16m7Q, d: 08:35:10, }"] [2025/04/06 08:25:37.050 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ c: bZ, ahRhZ: IBh7IpjWF4015SFKwUAVHoVCmXImvg4D78RZTlNUJlmR, d: 08:35:11, a: 615, b: 615, }"] [2025/04/06 08:25:37.150 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ c: Btfq, ahRhZ: iEOr126IxuLrAvOs3nICxqbmKT0HtdlknaW, d: 08:35:12, a: 616, b: 616, }"] [2025/04/06 08:25:37.250 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 617, b: 617, c: ut5Xfd, ahRhZ: myHk436MmLpVe7aED7e2IPCtr0PeyzK, d: 08:35:13, }"] wait process cdc.test exit for 2-th time... + set +x [Sun Apr 6 08:25:36 CST 2025] <<<<<< START cdc server in move_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.move_table.2367623678.out server --log-file /tmp/tidb_cdc_test/move_table/cdc2.log --log-level debug --data-dir /tmp/tidb_cdc_test/move_table/cdc_data2 --cluster-id default --addr 127.0.0.1:8301 + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2025/04/06 08:25:37.351 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ ahRhZ: eEoTcc1zOgLEjIR, d: 08:35:14, a: 618, b: 618, c: GjKqsfx8UX, }"] [2025/04/06 08:25:37.451 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ c: YagECSN4d, ahRhZ: qEObkNPGNtzCXRd1i, d: 08:35:15, a: 619, b: 619, }"] [2025/04/06 08:25:37.551 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ d: 08:35:16, a: 620, b: 620, c: o, ahRhZ: VgauzvAfkJ, }"] [row2="{ ahRhZ: LWRvZOsg5yZDgosUqvQJZzTkOh5vCHX4E4, d: 08:35:16, a: 620, b: 620, c: DDyG, }"] pass test: delete changefeed apis pass test: capture apis pass test: processor apis nonempty select id, val from test.availability1 where id=2 and val=22 run task successfully } % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 182 100 182 0 0 165k 0 --:--:-- --:--:-- --:--:-- 177k { "version": "v9.0.0-beta.1-9-g4c631d595", "git_hash": "4c631d5951854ac91765b729fe3849d8e8d5ed8a", "id": "0aa13c90-cd20-4c76-8448-46175271ef00", "pid": 11675, "is_owner": true [2025/04/06 08:25:37.651 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ c: o, ahRhZ: VgauzvAfkJ, d: 08:35:17, a: 621, b: 621, }"] [row2="{ b: 621, c: Va6Q0Hd2, ahRhZ: dZU0D9hLOxplju09BWkEFpoGLNUr, d: 08:35:17, a: 621, }"] [2025/04/06 08:25:37.751 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ d: 08:35:18, a: 622, b: 622, c: o, ahRhZ: VgauzvAfkJ, }"] [row2="{ b: 622, c: E, ahRhZ: 9r9FWMmXhuX2nwkiY1KMFskPOBpkILGOs, d: 08:35:18, a: 622, }"] cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Sun Apr 6 08:25:37 CST 2025] <<<<<< run test case multi_rocks success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) }wait process cdc.test exit for 1-th time... [2025/04/06 08:25:37.852 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ d: 08:35:19, a: 623, b: 623, c: o, ahRhZ: VgauzvAfkJ, }"] [row2="{ ahRhZ: 0pl5LgNkxWWSsJ5ZQXdQg9seyQVxLXMs9OWuuJ7, d: 08:35:19, a: 623, b: 623, c: sZLM, }"] [2025/04/06 08:25:37.952 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ c: o, ahRhZ: VgauzvAfkJ, d: 08:40:36, a: 624, b: 624, }"] [row2="{ ahRhZ: mBAiPV, d: 08:40:36, a: 624, b: 624, c: 7ZPO6, }"] [2025/04/06 08:25:38.052 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 625, b: 625, c: o, ahRhZ: VgauzvAfkJ, d: 08:35:21, }"] [row2="{ a: 625, b: 625, c: Ou0, ahRhZ: neNMezA7q3Q, d: 08:35:21, }"] Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table changefeed_auto_stop_4.usertable exists check diff successfully wait process cdc.test exit for 1-th time... [2025/04/06 08:25:38.152 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ b: 626, c: o, ahRhZ: VgauzvAfkJ, d: 08:35:22, a: 626, }"] [row2="{ a: 626, b: 626, c: JYZlO, ahRhZ: MvHu3uI9sMwc1E6VmrE9W0pbuqJ8c, d: 08:35:22, }"] [2025/04/06 08:25:38.252 +08:00] [WARN] [diff.go:937] ["find different row"] [column=ahRhZ] [row1="{ a: 627, b: 627, c: o, ahRhZ: VgauzvAfkJ, d: 08:35:23, }"] [row2="{ d: 08:35:23, a: 627, b: 627, c: 7XK, ahRhZ: a9KeDWU3p9GVgDBV1P4yCYMe1HZUmDXaXYGeOlBjDORTk, }"] wait process cdc.test exit for 1-th time... [2025/04/06 08:25:38.352 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 628, b: 628, c: o, ahRhZ: VgauzvAfkJ, d: 08:35:24, }"] [row2="{ ahRhZ: Yjd3aeU, d: 08:35:24, a: 628, b: 628, c: wu, }"] [2025/04/06 08:25:38.453 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 629, b: 629, c: o, ahRhZ: VgauzvAfkJ, d: 08:35:25, }"] [row2="{ c: 5En8A, ahRhZ: csBXZQaPGaKW, d: 08:35:25, a: 629, b: 629, }"] [2025/04/06 08:25:38.553 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ b: 630, c: o, ahRhZ: VgauzvAfkJ, d: 08:35:27, a: 630, }"] [row2="{ d: 08:35:27, a: 630, b: 630, c: RBR, ahRhZ: dhsz49dMxUZMYCsLxTRRG2cTZZjimlJGECzRr, }"] wait process cdc.test exit for 2-th time... table ddl_attributes.finish_mark exists [2025/04/06 08:25:38.653 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ b: 631, c: o, ahRhZ: VgauzvAfkJ, d: 08:35:28, a: 631, }"] [row2="{ a: 631, b: 631, c: NUb5z0Ff, ahRhZ: lrKy3Jj1LAbHo7txu61, d: 08:35:28, }"] [2025/04/06 08:25:38.753 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 632, b: 632, c: o, ahRhZ: VgauzvAfkJ, d: 08:35:29, }"] [row2="{ ahRhZ: Oh4F2GoPFF9WNmeDgcIe276XIcQMox2E3I, d: 08:35:29, a: 632, b: 632, c: jSR6I3eEH, }"] wait process cdc.test exit for 2-th time... wait process cdc.test exit for 2-th time... check diff successfully [2025/04/06 08:25:38.853 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 633, b: 633, c: o, ahRhZ: VgauzvAfkJ, d: 08:35:30, }"] [row2="{ c: pS0a, ahRhZ: 6az4mblboFlbpn1trB6WMYjam5XaT, d: 08:35:30, a: 633, b: 633, }"] [2025/04/06 08:25:38.954 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ d: 08:35:31, a: 634, b: 634, c: o, ahRhZ: VgauzvAfkJ, }"] [row2="{ d: 08:35:31, a: 634, b: 634, c: esH, ahRhZ: l, }"] [2025/04/06 08:25:39.054 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 635, b: 635, c: o, ahRhZ: VgauzvAfkJ, d: 08:35:32, }"] [row2="{ b: 635, c: XeGtzLk, ahRhZ: 31pMWkG5N7msQBzgwFi, d: 08:35:32, a: 635, }"] cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Sun Apr 6 08:25:38 CST 2025] <<<<<< run test case cli_with_auth success! >>>>>> [2025/04/06 08:25:39.154 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ d: 08:35:34, a: 637, b: 637, c: o, ahRhZ: VgauzvAfkJ, }"] [row2="{ d: 08:35:34, a: 637, b: 637, c: amiCezdT, ahRhZ: DGDrlQFYJzU84gV7Ot5wEwFYuj3dDQOIgmUe6xQRbTH, }"] [2025/04/06 08:25:39.254 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 638, b: 638, c: o, ahRhZ: VgauzvAfkJ, d: 08:35:35, }"] [row2="{ d: 08:35:35, a: 638, b: 638, c: KYrfNSEfWW, ahRhZ: 7cqH85P, }"] cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit sql_check run sql_check, 127.0.0.1 run task successfully run test case test_stop_processor [Sun Apr 6 08:25:39 CST 2025] <<<<<< START cdc server in availability case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + GO_FAILPOINTS= + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.availability.66816683.out server --log-file /tmp/tidb_cdc_test/availability/cdctest_stop_processor.log --log-level debug --data-dir /tmp/tidb_cdc_test/availability/cdc_datatest_stop_processor --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Sun Apr 6 08:25:39 CST 2025] <<<<<< run test case changefeed_auto_stop success! >>>>>> wait process cdc.test exit for 1-th time... [2025/04/06 08:25:39.354 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 639, b: 639, c: o, ahRhZ: VgauzvAfkJ, d: 08:35:36, }"] [row2="{ a: 639, b: 639, c: 0bxkeGtO, ahRhZ: FWFL9DzvTTpTs1yBsD9NyXR2KfgvX, d: 08:35:36, }"] [2025/04/06 08:25:39.455 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 640, b: 640, c: o, ahRhZ: VgauzvAfkJ, d: 08:35:37, }"] [row2="{ a: 640, b: 640, c: 1K5i, ahRhZ: 3XOQ1W5WFLwXYCWzJWY0q573mEk8heYKdYs, d: 08:35:37, }"] [2025/04/06 08:25:39.555 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ d: 08:35:38, a: 641, b: 641, c: o, ahRhZ: VgauzvAfkJ, }"] [row2="{ a: 641, b: 641, c: iz, ahRhZ: EyhRgWAS6oV5zah1, d: 08:35:38, }"] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489989742764661549 TiDB Cluster ID. tikv_gc_leader_uuid 65821f46e44000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-z-4sl2f, pid:11066, start at 2025-04-06 08:25:38.978521897 +0800 CST m=+3.693156186 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:27:38.988 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:25:38.961 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:15:38.961 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489989742764661549 TiDB Cluster ID. tikv_gc_leader_uuid 65821f46e44000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-z-4sl2f, pid:11066, start at 2025-04-06 08:25:38.978521897 +0800 CST m=+3.693156186 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:27:38.988 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:25:38.961 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:15:38.961 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489989737460970938 TiDB Cluster ID. tikv_gc_leader_uuid 65821f46e000006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-z-4sl2f, pid:11122, start at 2025-04-06 08:25:38.953495884 +0800 CST m=+3.579183166 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:27:38.963 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:25:38.944 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:15:38.944 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/multi_cdc_cluster/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/multi_cdc_cluster/tiflash/log/error.log arg matches is ArgMatches { args: {"log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/multi_cdc_cluster/tiflash/log/proxy.log"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/multi_cdc_cluster/tiflash/db/proxy"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/multi_cdc_cluster/tiflash-proxy.toml"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [2025/04/06 08:25:39.655 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ ahRhZ: VgauzvAfkJ, d: 08:35:40, a: 643, b: 643, c: o, }"] [row2="{ ahRhZ: JhJMsc, d: 08:35:40, a: 643, b: 643, c: l, }"] [2025/04/06 08:25:39.755 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ b: 644, c: o, ahRhZ: VgauzvAfkJ, d: 08:41:49, a: 644, }"] [row2="{ a: 644, b: 644, c: pUdalr7, ahRhZ: pwUJPYDD9M5MqYkENZ, d: 08:35:41, }"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL [2025/04/06 08:25:39.855 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 645, b: 645, c: o, ahRhZ: VgauzvAfkJ, d: 08:35:42, }"] [row2="{ a: 645, b: 645, c: t, ahRhZ: VlWNcoKV9Y9tOFc1TUT, d: 08:35:42, }"] [2025/04/06 08:25:39.956 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 646, b: 646, c: o, ahRhZ: VgauzvAfkJ, d: 08:35:43, }"] [row2="{ ahRhZ: Ms7ITxAWpxHG, d: 08:35:43, a: 646, b: 646, c: ILoOc5df, }"] [2025/04/06 08:25:40.056 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ c: o, ahRhZ: VgauzvAfkJ, d: 08:35:44, a: 647, b: 647, }"] [row2="{ a: 647, b: 647, c: lLb, ahRhZ: dt5M5R1uzVfXs2W6sEyDAvgkzncXL9Gebz8sM5O, d: 08:35:44, }"] * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 06 Apr 2025 00:25:39 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/44202c40-4a17-4681-8452-63490f9b5e23 {UpstreamID:7489989674838134457 Namespace:default ID:44202c40-4a17-4681-8452-63490f9b5e23 SinkURI:mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1 CreateTime:2025-04-06 08:25:34.931675019 +0800 CST StartTs:457152692555612161 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0037ec360 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457152694691823675} {CheckpointTs:457152692961935376 MinTableBarrierTs:457152696042127371 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4322f23c-3086-4cf3-a91d-034ceb577f00 {"id":"4322f23c-3086-4cf3-a91d-034ceb577f00","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899137} /tidb/cdc/default/__cdc_meta__/capture/6f229d06-016c-4169-bab4-9209ad4184b8 {"id":"6f229d06-016c-4169-bab4-9209ad4184b8","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899131} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087cb9b505 6f229d06-016c-4169-bab4-9209ad4184b8 /tidb/cdc/default/__cdc_meta__/owner/223196087cb9b559 4322f23c-3086-4cf3-a91d-034ceb577f00 /tidb/cdc/default/default/changefeed/info/44202c40-4a17-4681-8452-63490f9b5e23 {"upstream-id":7489989674838134457,"namespace":"default","changefeed-id":"44202c40-4a17-4681-8452-63490f9b5e23","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2025-04-06T08:25:34.931675019+08:00","start-ts":457152692555612161,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457152694691823675} /tidb/cdc/default/default/changefeed/status/44202c40-4a17-4681-8452-63490f9b5e23 {"checkpoint-ts":457152692961935376,"min-table-barrier-ts":457152696042127371,"admin-job-type":0} /tidb/cdc/default/default/task/position/4322f23c-3086-4cf3-a91d-034ceb577f00/44202c40-4a17-4681-8452-63490f9b5e23 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/6f229d06-016c-4169-bab4-9209ad4184b8/44202c40-4a17-4681-8452-63490f9b5e23 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489989674838134457 {"id":7489989674838134457,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: changefeedID: default/44202c40-4a17-4681-8452-63490f9b5e23 {UpstreamID:7489989674838134457 Namespace:default ID:44202c40-4a17-4681-8452-63490f9b5e23 SinkURI:mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1 CreateTime:2025-04-06 08:25:34.931675019 +0800 CST StartTs:457152692555612161 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0037ec360 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457152694691823675} {CheckpointTs:457152692961935376 MinTableBarrierTs:457152696042127371 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4322f23c-3086-4cf3-a91d-034ceb577f00 {"id":"4322f23c-3086-4cf3-a91d-034ceb577f00","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899137} /tidb/cdc/default/__cdc_meta__/capture/6f229d06-016c-4169-bab4-9209ad4184b8 {"id":"6f229d06-016c-4169-bab4-9209ad4184b8","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899131} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087cb9b505 6f229d06-016c-4169-bab4-9209ad4184b8 /tidb/cdc/default/__cdc_meta__/owner/223196087cb9b559 4322f23c-3086-4cf3-a91d-034ceb577f00 /tidb/cdc/default/default/changefeed/info/44202c40-4a17-4681-8452-63490f9b5e23 {"upstream-id":7489989674838134457,"namespace":"default","changefeed-id":"44202c40-4a17-4681-8452-63490f9b5e23","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2025-04-06T08:25:34.931675019+08:00","start-ts":457152692555612161,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457152694691823675} /tidb/cdc/default/default/changefeed/status/44202c40-4a17-4681-8452-63490f9b5e23 {"checkpoint-ts":457152692961935376,"min-table-barrier-ts":457152696042127371,"admin-job-type":0} /tidb/cdc/default/default/task/position/4322f23c-3086-4cf3-a91d-034ceb577f00/44202c40-4a17-4681-8452-63490f9b5e23 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/6f229d06-016c-4169-bab4-9209ad4184b8/44202c40-4a17-4681-8452-63490f9b5e23 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489989674838134457 {"id":7489989674838134457,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/44202c40-4a17-4681-8452-63490f9b5e23 {UpstreamID:7489989674838134457 Namespace:default ID:44202c40-4a17-4681-8452-63490f9b5e23 SinkURI:mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1 CreateTime:2025-04-06 08:25:34.931675019 +0800 CST StartTs:457152692555612161 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0037ec360 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457152694691823675} {CheckpointTs:457152692961935376 MinTableBarrierTs:457152696042127371 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4322f23c-3086-4cf3-a91d-034ceb577f00 {"id":"4322f23c-3086-4cf3-a91d-034ceb577f00","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899137} /tidb/cdc/default/__cdc_meta__/capture/6f229d06-016c-4169-bab4-9209ad4184b8 {"id":"6f229d06-016c-4169-bab4-9209ad4184b8","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899131} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087cb9b505 6f229d06-016c-4169-bab4-9209ad4184b8 /tidb/cdc/default/__cdc_meta__/owner/223196087cb9b559 4322f23c-3086-4cf3-a91d-034ceb577f00 /tidb/cdc/default/default/changefeed/info/44202c40-4a17-4681-8452-63490f9b5e23 {"upstream-id":7489989674838134457,"namespace":"default","changefeed-id":"44202c40-4a17-4681-8452-63490f9b5e23","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2025-04-06T08:25:34.931675019+08:00","start-ts":457152692555612161,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457152694691823675} /tidb/cdc/default/default/changefeed/status/44202c40-4a17-4681-8452-63490f9b5e23 {"checkpoint-ts":457152692961935376,"min-table-barrier-ts":457152696042127371,"admin-job-type":0} /tidb/cdc/default/default/task/position/4322f23c-3086-4cf3-a91d-034ceb577f00/44202c40-4a17-4681-8452-63490f9b5e23 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/6f229d06-016c-4169-bab4-9209ad4184b8/44202c40-4a17-4681-8452-63490f9b5e23 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489989674838134457 {"id":7489989674838134457,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Sun Apr 6 08:25:39 CST 2025] <<<<<< START cdc server in move_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.move_table.2374023742.out server --log-file /tmp/tidb_cdc_test/move_table/cdc3.log --log-level debug --data-dir /tmp/tidb_cdc_test/move_table/cdc_data3 --cluster-id default --addr 127.0.0.1:8302 + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8302; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 wait process cdc.test exit for 2-th time... [2025/04/06 08:25:40.156 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 650, b: 650, c: o, ahRhZ: VgauzvAfkJ, d: 08:35:48, }"] [row2="{ ahRhZ: 7cmxZOWSIjjYHOFkDXyMXTEjcJubJ, d: 08:35:48, a: 650, b: 650, c: FRK59UKN, }"] [2025/04/06 08:25:40.256 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ c: o, ahRhZ: VgauzvAfkJ, d: 08:35:49, a: 651, b: 651, }"] [row2="{ c: x, ahRhZ: kL64uORSRsOFwsKxu2LqnkFhZWEKsKHQ2j, d: 08:35:49, a: 651, b: 651, }"] cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [2025/04/06 08:25:40.356 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ d: 08:35:50, a: 652, b: 652, c: o, ahRhZ: VgauzvAfkJ, }"] [row2="{ a: 652, b: 652, c: gjD7CTXFM, ahRhZ: fSE, d: 08:35:50, }"] [2025/04/06 08:25:40.457 +08:00] [WARN] [diff.go:937] ["find different row"] [column=ahRhZ] [row1="{ a: 653, b: 653, c: o, ahRhZ: VgauzvAfkJ, d: 08:35:51, }"] [row2="{ c: xmp6QZVYs, ahRhZ: UveeCWSDICqz, d: 08:35:51, a: 653, b: 653, }"] [2025/04/06 08:25:40.557 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 654, b: 654, c: o, ahRhZ: VgauzvAfkJ, d: 08:35:52, }"] [row2="{ b: 654, c: h96XMNl, ahRhZ: cpn21ldH0H3WgjK1NDgSbbYUD6GXnk0seEoMNXSGiS, d: 08:35:52, a: 654, }"] [Sun Apr 6 08:25:40 CST 2025] <<<<<< run test case ddl_attributes success! >>>>>> Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2025/04/06 08:25:40.657 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 655, b: 655, c: o, ahRhZ: VgauzvAfkJ, d: 08:35:53, }"] [row2="{ d: 08:35:53, a: 655, b: 655, c: V1oQ, ahRhZ: lq6e7FUVTFj3ViSZni5EooHAhxN5hbrVIBI1U4, }"] [2025/04/06 08:25:40.757 +08:00] [WARN] [diff.go:937] ["find different row"] [column=ahRhZ] [row1="{ ahRhZ: VgauzvAfkJ, d: 08:35:54, a: 656, b: 656, c: o, }"] [row2="{ b: 656, c: kc, ahRhZ: d1uqSARoInG, d: 08:35:54, a: 656, }"] pass test: owner apis pass test: set log level [2025/04/06 08:25:40.857 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ d: 08:35:55, a: 657, b: 657, c: o, ahRhZ: VgauzvAfkJ, }"] [row2="{ a: 657, b: 657, c: 7MYY, ahRhZ: e9WVK5fYiCzvV8OUh44JvQlGNnI5Cnm, d: 08:35:55, }"] [2025/04/06 08:25:40.958 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 658, b: 658, c: o, ahRhZ: VgauzvAfkJ, d: 08:35:56, }"] [row2="{ a: 658, b: 658, c: lMz8W8H, ahRhZ: rWC5ZXh4BPbSQGQn, d: 08:35:56, }"] [2025/04/06 08:25:41.058 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ c: o, ahRhZ: VgauzvAfkJ, d: 08:35:57, a: 659, b: 659, }"] [row2="{ b: 659, c: zUuOZlG2s, ahRhZ: 5BXjX4xPC, d: 08:35:57, a: 659, }"] + cleanup_process cdc.test check_redo_resolved_ts cb6b1b7f-ab29-4852-a097-f142c1ff4ff0 457152696404934662 file:///tmp/tidb_cdc_test/consistent_replicate_storage_file/redo /tmp/tidb_cdc_test/consistent_replicate_storage_file/cdc_data/redo/cb6b1b7f-ab29-4852-a097-f142c1ff4ff0/meta global resolved ts 457152690690195464 not forward to 457152696404934662 run task failed 1-th time, retry later [2025/04/06 08:25:41.158 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 660, b: 660, c: o, ahRhZ: VgauzvAfkJ, d: 08:35:58, }"] [row2="{ b: 660, c: hX, ahRhZ: iv9B, d: 08:35:58, a: 660, }"] [2025/04/06 08:25:41.258 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 661, b: 661, c: o, ahRhZ: VgauzvAfkJ, d: 08:35:59, }"] [row2="{ a: 661, b: 661, c: he6g7hFG, ahRhZ: tcjziIXonOM3XTJS1m2oQQknr2, d: 08:35:59, }"] [2025/04/06 08:25:41.358 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 662, b: 662, c: o, ahRhZ: VgauzvAfkJ, d: 08:36:00, }"] [row2="{ b: 662, c: PukcgACKCC, ahRhZ: FzLOm35nCzp8S1uahzZHJgattwVYgfGInV0fWuPISz57c, d: 08:36:00, a: 662, }"] [2025/04/06 08:25:41.458 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ d: 08:36:01, a: 663, b: 663, c: o, ahRhZ: VgauzvAfkJ, }"] [row2="{ ahRhZ: uM6qrfI3JHjjIVaK6YdPZyU7Gght6, d: 08:36:01, a: 663, b: 663, c: lIrDbhI7, }"] [2025/04/06 08:25:41.559 +08:00] [WARN] [diff.go:937] ["find different row"] [column=ahRhZ] [row1="{ a: 664, b: 664, c: o, ahRhZ: VgauzvAfkJ, d: 08:36:02, }"] [row2="{ a: 664, b: 664, c: P8t0RUZ, ahRhZ: s0b2v3fpZCfz0XPJoeSpP, d: 08:36:02, }"] wait process cdc.test exit for 1-th time... [2025/04/06 08:25:41.659 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 665, b: 665, c: o, ahRhZ: VgauzvAfkJ, d: 08:36:03, }"] [row2="{ c: v, ahRhZ: knZTom4JqBLbevfc2qv0wNZhspOFZGkHg, d: 08:36:03, a: 665, b: 665, }"] [2025/04/06 08:25:41.759 +08:00] [WARN] [diff.go:937] ["find different row"] [column=ahRhZ] [row1="{ ahRhZ: VgauzvAfkJ, d: 08:36:04, a: 666, b: 666, c: o, }"] [row2="{ d: 08:36:04, a: 666, b: 666, c: kjiv, ahRhZ: 6DIEnSSGI9TiJS, }"] [2025/04/06 08:25:41.859 +08:00] [WARN] [diff.go:937] ["find different row"] [column=ahRhZ] [row1="{ c: o, ahRhZ: VgauzvAfkJ, d: 08:36:05, a: 667, b: 667, }"] [row2="{ a: 667, b: 667, c: ZN2, ahRhZ: xNQ3GJ1KAeWQ5jE, d: 08:36:05, }"] [2025/04/06 08:25:41.959 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ d: 08:36:06, a: 668, b: 668, c: o, ahRhZ: VgauzvAfkJ, }"] [row2="{ d: 08:36:06, a: 668, b: 668, c: qjVLgdWtml, ahRhZ: uQVpyAZIMM6C3DATbr9PUGUYT6Zw6utwsG, }"] [2025/04/06 08:25:42.060 +08:00] [WARN] [diff.go:937] ["find different row"] [column=ahRhZ] [row1="{ a: 669, b: 669, c: o, ahRhZ: VgauzvAfkJ, d: 08:36:07, }"] [row2="{ a: 669, b: 669, c: oVCW5, ahRhZ: xAk1rmGtZD4NkqVHipwMLikCCH, d: 08:36:07, }"] + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_cdc_cluster.cli.12353.out cli tso query --pd=http://127.0.0.1:2379 wait process cdc.test exit for 2-th time... [2025/04/06 08:25:42.160 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ c: o, ahRhZ: VgauzvAfkJ, d: 08:36:08, a: 670, b: 670, }"] [row2="{ a: 670, b: 670, c: vlknSbwwt6, ahRhZ: JXWCDBP2kfeEh7ennrwxZyXXLXzGKyvXUmGa, d: 08:36:08, }"] [2025/04/06 08:25:42.260 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ b: 671, c: o, ahRhZ: VgauzvAfkJ, d: 08:36:09, a: 671, }"] [row2="{ b: 671, c: U, ahRhZ: 6AOwxwcuVFeY5LddYO, d: 08:36:09, a: 671, }"] [2025/04/06 08:25:42.360 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ c: o, ahRhZ: VgauzvAfkJ, d: 08:36:10, a: 672, b: 672, }"] [row2="{ b: 672, c: CFvO, ahRhZ: rkfr9LwS5tAljF2R3eWV7p, d: 08:36:10, a: 672, }"] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489989357300557951 TiDB Cluster ID. tikv_gc_leader_uuid 65821ef08600007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-j-shst4, pid:9157, start at 2025-04-06 08:24:10.546273365 +0800 CST m=+3.817043453 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:26:10.556 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:24:10.520 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:14:10.520 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489989357300557951 TiDB Cluster ID. tikv_gc_leader_uuid 65821ef08600007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-j-shst4, pid:9157, start at 2025-04-06 08:24:10.546273365 +0800 CST m=+3.817043453 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:26:10.556 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:24:10.520 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:14:10.520 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489989362460687017 TiDB Cluster ID. tikv_gc_leader_uuid 65821ef09fc0002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-j-shst4, pid:9225, start at 2025-04-06 08:24:10.627926606 +0800 CST m=+3.832542938 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:26:10.644 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:24:10.623 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:14:10.623 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed + run_sql 'set global sql_mode='\''ANSI_QUOTES'\'';' 127.0.0.1 4000 + run_sql 'set global sql_mode='\''ANSI_QUOTES'\'';' 127.0.0.1 3306 wait process cdc.test exit for 3-th time... process cdc.test already exit ++ date + echo '[Sun Apr 6 08:25:42 CST 2025] <<<<<< run test case api_v2 success! >>>>>>' [Sun Apr 6 08:25:42 CST 2025] <<<<<< run test case api_v2 success! >>>>>> + stop_tidb_cluster [2025/04/06 08:25:42.460 +08:00] [WARN] [diff.go:937] ["find different row"] [column=ahRhZ] [row1="{ d: 08:36:11, a: 673, b: 673, c: o, ahRhZ: VgauzvAfkJ, }"] [row2="{ a: 673, b: 673, c: EY9Hit4zF, ahRhZ: mNNo, d: 08:36:11, }"] [2025/04/06 08:25:42.561 +08:00] [WARN] [diff.go:937] ["find different row"] [column=ahRhZ] [row1="{ d: 08:36:12, a: 674, b: 674, c: o, ahRhZ: VgauzvAfkJ, }"] [row2="{ a: 674, b: 674, c: b6wi, ahRhZ: bw5a7ux, d: 08:36:12, }"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 06 Apr 2025 00:25:42 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/e684662d-022f-4a79-9ef3-0a59dba96193 {UpstreamID:7489988628380945915 Namespace:default ID:e684662d-022f-4a79-9ef3-0a59dba96193 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-06 08:21:28.032207553 +0800 CST StartTs:457152628763131907 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc004476990 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457152630100066306} {CheckpointTs:457152696566415365 MinTableBarrierTs:457152696566415365 AdminJobType:noop} span: {tableID:112, startKey:7480000000000000ff705f720000000000fa, endKey:7480000000000000ff705f730000000000fa}, resolvedTs: 457152696579522561, checkpointTs: 457152696566415365, state: Replicating span: {tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}, resolvedTs: 457152696566415366, checkpointTs: 457152696566415365, state: Replicating span: {tableID:116, startKey:7480000000000000ff745f720000000000fa, endKey:7480000000000000ff745f730000000000fa}, resolvedTs: 457152696566415365, checkpointTs: 457152696566415365, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b2fc3695-c374-49af-865d-796d9a81d5d6 {"id":"b2fc3695-c374-49af-865d-796d9a81d5d6","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899139} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 4 /tidb/cdc/default/__cdc_meta__/owner/22319608790550e9 b2fc3695-c374-49af-865d-796d9a81d5d6 /tidb/cdc/default/default/changefeed/info/e684662d-022f-4a79-9ef3-0a59dba96193 {"upstream-id":7489988628380945915,"namespace":"default","changefeed-id":"e684662d-022f-4a79-9ef3-0a59dba96193","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-06T08:21:28.032207553+08:00","start-ts":457152628763131907,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457152630100066306} /tidb/cdc/default/default/changefeed/status/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":457152696566415365,"min-table-barrier-ts":457152696566415365,"admin-job-type":0} /tidb/cdc/default/default/task/position/b2fc3695-c374-49af-865d-796d9a81d5d6/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489988628380945915 {"id":7489988628380945915,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/e684662d-022f-4a79-9ef3-0a59dba96193 {UpstreamID:7489988628380945915 Namespace:default ID:e684662d-022f-4a79-9ef3-0a59dba96193 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-06 08:21:28.032207553 +0800 CST StartTs:457152628763131907 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc004476990 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457152630100066306} {CheckpointTs:457152696566415365 MinTableBarrierTs:457152696566415365 AdminJobType:noop} span: {tableID:112, startKey:7480000000000000ff705f720000000000fa, endKey:7480000000000000ff705f730000000000fa}, resolvedTs: 457152696579522561, checkpointTs: 457152696566415365, state: Replicating span: {tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}, resolvedTs: 457152696566415366, checkpointTs: 457152696566415365, state: Replicating span: {tableID:116, startKey:7480000000000000ff745f720000000000fa, endKey:7480000000000000ff745f730000000000fa}, resolvedTs: 457152696566415365, checkpointTs: 457152696566415365, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b2fc3695-c374-49af-865d-796d9a81d5d6 {"id":"b2fc3695-c374-49af-865d-796d9a81d5d6","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899139} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 4 /tidb/cdc/default/__cdc_meta__/owner/22319608790550e9 b2fc3695-c374-49af-865d-796d9a81d5d6 /tidb/cdc/default/default/changefeed/info/e684662d-022f-4a79-9ef3-0a59dba96193 {"upstream-id":7489988628380945915,"namespace":"default","changefeed-id":"e684662d-022f-4a79-9ef3-0a59dba96193","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-06T08:21:28.032207553+08:00","start-ts":457152628763131907,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457152630100066306} /tidb/cdc/default/default/changefeed/status/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":457152696566415365,"min-table-barrier-ts":457152696566415365,"admin-job-type":0} /tidb/cdc/default/default/task/position/b2fc3695-c374-49af-865d-796d9a81d5d6/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489988628380945915 {"id":7489988628380945915,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/e684662d-022f-4a79-9ef3-0a59dba96193 {UpstreamID:7489988628380945915 Namespace:default ID:e684662d-022f-4a79-9ef3-0a59dba96193 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-06 08:21:28.032207553 +0800 CST StartTs:457152628763131907 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc004476990 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457152630100066306} {CheckpointTs:457152696566415365 MinTableBarrierTs:457152696566415365 AdminJobType:noop} span: {tableID:112, startKey:7480000000000000ff705f720000000000fa, endKey:7480000000000000ff705f730000000000fa}, resolvedTs: 457152696579522561, checkpointTs: 457152696566415365, state: Replicating span: {tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}, resolvedTs: 457152696566415366, checkpointTs: 457152696566415365, state: Replicating span: {tableID:116, startKey:7480000000000000ff745f720000000000fa, endKey:7480000000000000ff745f730000000000fa}, resolvedTs: 457152696566415365, checkpointTs: 457152696566415365, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b2fc3695-c374-49af-865d-796d9a81d5d6 {"id":"b2fc3695-c374-49af-865d-796d9a81d5d6","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899139} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 4 /tidb/cdc/default/__cdc_meta__/owner/22319608790550e9 b2fc3695-c374-49af-865d-796d9a81d5d6 /tidb/cdc/default/default/changefeed/info/e684662d-022f-4a79-9ef3-0a59dba96193 {"upstream-id":7489988628380945915,"namespace":"default","changefeed-id":"e684662d-022f-4a79-9ef3-0a59dba96193","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-06T08:21:28.032207553+08:00","start-ts":457152628763131907,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457152630100066306} /tidb/cdc/default/default/changefeed/status/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":457152696566415365,"min-table-barrier-ts":457152696566415365,"admin-job-type":0} /tidb/cdc/default/default/task/position/b2fc3695-c374-49af-865d-796d9a81d5d6/e684662d-022f-4a79-9ef3-0a59dba96193 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489988628380945915 {"id":7489988628380945915,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x cdc.test cli capture list 2>&1 | grep '"is-owner": true' ++ run_cdc_cli_tso_query 127.0.0.1 2379 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sql_mode.cli.14503.out cli tso query --pd=http://127.0.0.1:2379 [2025/04/06 08:25:42.661 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ d: 08:36:13, a: 675, b: 675, c: o, ahRhZ: VgauzvAfkJ, }"] [row2="{ b: 675, c: aj, ahRhZ: dJwOItTE2ITrwpgG1mCBaHe2fVoiJxsmu1tJzfb, d: 08:36:13, a: 675, }"] [2025/04/06 08:25:42.761 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ ahRhZ: VgauzvAfkJ, d: 08:36:14, a: 676, b: 676, c: o, }"] [row2="{ a: 676, b: 676, c: e4YvOsonp, ahRhZ: xLOYPhave6MXq7tNz, d: 08:36:14, }"] [2025/04/06 08:25:42.861 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 677, b: 677, c: o, ahRhZ: VgauzvAfkJ, d: 08:36:15, }"] [row2="{ a: 677, b: 677, c: ibcLf, ahRhZ: qywwGtlbus8nHeinAt5Uv, d: 08:36:15, }"] [2025/04/06 08:25:42.961 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 678, b: 678, c: o, ahRhZ: VgauzvAfkJ, d: 08:36:16, }"] [row2="{ a: 678, b: 678, c: qe7SI2D4, ahRhZ: PxHMhi5PDxQYIP2unwQTjA6L3lS, d: 08:36:16, }"] [2025/04/06 08:25:43.062 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ d: 08:36:17, a: 679, b: 679, c: o, ahRhZ: VgauzvAfkJ, }"] [row2="{ a: 679, b: 679, c: GNJAF, ahRhZ: c0rQfH, d: 08:36:17, }"] check_redo_resolved_ts cb6b1b7f-ab29-4852-a097-f142c1ff4ff0 457152696404934662 file:///tmp/tidb_cdc_test/consistent_replicate_storage_file/redo /tmp/tidb_cdc_test/consistent_replicate_storage_file/cdc_data/redo/cb6b1b7f-ab29-4852-a097-f142c1ff4ff0/meta + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8302 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8302 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 06 Apr 2025 00:25:43 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/44202c40-4a17-4681-8452-63490f9b5e23 {UpstreamID:7489989674838134457 Namespace:default ID:44202c40-4a17-4681-8452-63490f9b5e23 SinkURI:mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1 CreateTime:2025-04-06 08:25:34.931675019 +0800 CST StartTs:457152692555612161 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003e60900 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457152694691823675} {CheckpointTs:457152693227225201 MinTableBarrierTs:457152696845074437 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/33241d46-9f8f-4c49-9223-9ee69e44e36f {"id":"33241d46-9f8f-4c49-9223-9ee69e44e36f","address":"127.0.0.1:8302","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899141} /tidb/cdc/default/__cdc_meta__/capture/4322f23c-3086-4cf3-a91d-034ceb577f00 {"id":"4322f23c-3086-4cf3-a91d-034ceb577f00","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899137} /tidb/cdc/default/__cdc_meta__/capture/6f229d06-016c-4169-bab4-9209ad4184b8 {"id":"6f229d06-016c-4169-bab4-9209ad4184b8","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899131} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087cb9b505 6f229d06-016c-4169-bab4-9209ad4184b8 /tidb/cdc/default/__cdc_meta__/owner/223196087cb9b559 4322f23c-3086-4cf3-a91d-034ceb577f00 /tidb/cdc/default/__cdc_meta__/owner/223196087cb9b5a0 33241d46-9f8f-4c49-9223-9ee69e44e36f /tidb/cdc/default/default/changefeed/info/44202c40-4a17-4681-8452-63490f9b5e23 {"upstream-id":7489989674838134457,"namespace":"default","changefeed-id":"44202c40-4a17-4681-8452-63490f9b5e23","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2025-04-06T08:25:34.931675019+08:00","start-ts":457152692555612161,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457152694691823675} /tidb/cdc/default/default/changefeed/status/44202c40-4a17-4681-8452-63490f9b5e23 {"checkpoint-ts":457152693227225201,"min-table-barrier-ts":457152696845074437,"admin-job-type":0} /tidb/cdc/default/default/task/position/33241d46-9f8f-4c49-9223-9ee69e44e36f/44202c40-4a17-4681-8452-63490f9b5e23 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/4322f23c-3086-4cf3-a91d-034ceb577f00/44202c40-4a17-4681-8452-63490f9b5e23 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/6f229d06-016c-4169-bab4-9209ad4184b8/44202c40-4a17-4681-8452-63490f9b5e23 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489989674838134457 {"id":7489989674838134457,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: changefeedID: default/44202c40-4a17-4681-8452-63490f9b5e23 {UpstreamID:7489989674838134457 Namespace:default ID:44202c40-4a17-4681-8452-63490f9b5e23 SinkURI:mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1 CreateTime:2025-04-06 08:25:34.931675019 +0800 CST StartTs:457152692555612161 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003e60900 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457152694691823675} {CheckpointTs:457152693227225201 MinTableBarrierTs:457152696845074437 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/33241d46-9f8f-4c49-9223-9ee69e44e36f {"id":"33241d46-9f8f-4c49-9223-9ee69e44e36f","address":"127.0.0.1:8302","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899141} /tidb/cdc/default/__cdc_meta__/capture/4322f23c-3086-4cf3-a91d-034ceb577f00 {"id":"4322f23c-3086-4cf3-a91d-034ceb577f00","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899137} /tidb/cdc/default/__cdc_meta__/capture/6f229d06-016c-4169-bab4-9209ad4184b8 {"id":"6f229d06-016c-4169-bab4-9209ad4184b8","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899131} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087cb9b505 6f229d06-016c-4169-bab4-9209ad4184b8 /tidb/cdc/default/__cdc_meta__/owner/223196087cb9b559 4322f23c-3086-4cf3-a91d-034ceb577f00 /tidb/cdc/default/__cdc_meta__/owner/223196087cb9b5a0 33241d46-9f8f-4c49-9223-9ee69e44e36f /tidb/cdc/default/default/changefeed/info/44202c40-4a17-4681-8452-63490f9b5e23 {"upstream-id":7489989674838134457,"namespace":"default","changefeed-id":"44202c40-4a17-4681-8452-63490f9b5e23","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2025-04-06T08:25:34.931675019+08:00","start-ts":457152692555612161,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457152694691823675} /tidb/cdc/default/default/changefeed/status/44202c40-4a17-4681-8452-63490f9b5e23 {"checkpoint-ts":457152693227225201,"min-table-barrier-ts":457152696845074437,"admin-job-type":0} /tidb/cdc/default/default/task/position/33241d46-9f8f-4c49-9223-9ee69e44e36f/44202c40-4a17-4681-8452-63490f9b5e23 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/4322f23c-3086-4cf3-a91d-034ceb577f00/44202c40-4a17-4681-8452-63490f9b5e23 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/6f229d06-016c-4169-bab4-9209ad4184b8/44202c40-4a17-4681-8452-63490f9b5e23 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489989674838134457 {"id":7489989674838134457,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/44202c40-4a17-4681-8452-63490f9b5e23 {UpstreamID:7489989674838134457 Namespace:default ID:44202c40-4a17-4681-8452-63490f9b5e23 SinkURI:mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1 CreateTime:2025-04-06 08:25:34.931675019 +0800 CST StartTs:457152692555612161 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003e60900 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457152694691823675} {CheckpointTs:457152693227225201 MinTableBarrierTs:457152696845074437 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/33241d46-9f8f-4c49-9223-9ee69e44e36f {"id":"33241d46-9f8f-4c49-9223-9ee69e44e36f","address":"127.0.0.1:8302","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899141} /tidb/cdc/default/__cdc_meta__/capture/4322f23c-3086-4cf3-a91d-034ceb577f00 {"id":"4322f23c-3086-4cf3-a91d-034ceb577f00","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899137} /tidb/cdc/default/__cdc_meta__/capture/6f229d06-016c-4169-bab4-9209ad4184b8 {"id":"6f229d06-016c-4169-bab4-9209ad4184b8","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899131} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087cb9b505 6f229d06-016c-4169-bab4-9209ad4184b8 /tidb/cdc/default/__cdc_meta__/owner/223196087cb9b559 4322f23c-3086-4cf3-a91d-034ceb577f00 /tidb/cdc/default/__cdc_meta__/owner/223196087cb9b5a0 33241d46-9f8f-4c49-9223-9ee69e44e36f /tidb/cdc/default/default/changefeed/info/44202c40-4a17-4681-8452-63490f9b5e23 {"upstream-id":7489989674838134457,"namespace":"default","changefeed-id":"44202c40-4a17-4681-8452-63490f9b5e23","sink-uri":"mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1","create-time":"2025-04-06T08:25:34.931675019+08:00","start-ts":457152692555612161,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457152694691823675} /tidb/cdc/default/default/changefeed/status/44202c40-4a17-4681-8452-63490f9b5e23 {"checkpoint-ts":457152693227225201,"min-table-barrier-ts":457152696845074437,"admin-job-type":0} /tidb/cdc/default/default/task/position/33241d46-9f8f-4c49-9223-9ee69e44e36f/44202c40-4a17-4681-8452-63490f9b5e23 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/4322f23c-3086-4cf3-a91d-034ceb577f00/44202c40-4a17-4681-8452-63490f9b5e23 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/6f229d06-016c-4169-bab4-9209ad4184b8/44202c40-4a17-4681-8452-63490f9b5e23 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489989674838134457 {"id":7489989674838134457,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x table move_table.usertable exists [2025/04/06 08:25:43.162 +08:00] [WARN] [diff.go:937] ["find different row"] [column=ahRhZ] [row1="{ a: 680, b: 680, c: o, ahRhZ: VgauzvAfkJ, d: 08:36:18, }"] [row2="{ b: 680, c: O, ahRhZ: goEznX7aIwVbnxzqUvMRcigfchi5mcaOFxVLagdsLucev, d: 08:36:18, a: 680, }"] [2025/04/06 08:25:43.263 +08:00] [WARN] [diff.go:937] ["find different row"] [column=ahRhZ] [row1="{ b: 681, c: o, ahRhZ: VgauzvAfkJ, d: 08:36:19, a: 681, }"] [row2="{ a: 681, b: 681, c: QD5, ahRhZ: U2bInqh9ryMKS1Fv, d: 08:36:19, }"] [2025/04/06 08:25:43.363 +08:00] [WARN] [diff.go:937] ["find different row"] [column=ahRhZ] [row1="{ ahRhZ: VgauzvAfkJ, d: 08:36:20, a: 682, b: 682, c: o, }"] [row2="{ d: 08:36:20, a: 682, b: 682, c: af, ahRhZ: YGvh1GIYO5YNKQRTqB1a5y2zn8hrrqu, }"] global resolved ts 457152690690195464 not forward to 457152696404934662 run task failed 2-th time, retry later go: downloading github.com/benbjohnson/clock v1.1.0 go: downloading github.com/xdg/scram v1.0.5 go: downloading github.com/IBM/sarama v1.41.2 go: downloading go.etcd.io/etcd/server/v3 v3.5.12 go: downloading github.com/apache/pulsar-client-go v0.11.0 go: downloading github.com/aws/aws-sdk-go-v2 v1.19.1 go: downloading github.com/tinylib/msgp v1.1.6 go: downloading github.com/KimMachineGun/automemlimit v0.2.4 go: downloading github.com/go-mysql-org/go-mysql v1.9.1 go: downloading gorm.io/gorm v1.25.11 [2025/04/06 08:25:43.463 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 683, b: 683, c: o, ahRhZ: VgauzvAfkJ, d: 08:36:21, }"] [row2="{ b: 683, c: 7OvI5f, ahRhZ: werdRBXaD4iGvwFUZBcIioNzrE0OshOIFrjHHlSOD, d: 08:36:21, a: 683, }"] [2025/04/06 08:25:43.563 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ ahRhZ: VgauzvAfkJ, d: 08:36:22, a: 684, b: 684, c: o, }"] [row2="{ a: 684, b: 684, c: M, ahRhZ: RDFuGbkkHCmcyZs7i, d: 08:36:22, }"] + set +x + tso='457152696683593731 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 457152696683593731 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x go: downloading github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 go: downloading github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 go: downloading github.com/gavv/monotime v0.0.0-20190418164738-30dba4353424 go: downloading github.com/edwingeng/deque v0.0.0-20191220032131-8596380dee17 go: downloading github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5 go: downloading github.com/gin-gonic/gin v1.9.1 go: downloading github.com/stretchr/objx v0.5.2 go: downloading github.com/containerd/cgroups v1.0.4 go: downloading github.com/xdg/stringprep v1.0.3 go: downloading github.com/philhofer/fwd v1.1.1 [2025/04/06 08:25:43.663 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ c: o, ahRhZ: VgauzvAfkJ, d: 08:36:23, a: 685, b: 685, }"] [row2="{ a: 685, b: 685, c: DowouCGi3T, ahRhZ: AsgXGkvWQqAl8oSrjVecLarjl, d: 08:36:23, }"] [2025/04/06 08:25:43.764 +08:00] [WARN] [diff.go:940] ["target had superfluous data"] [row="{ ahRhZ: xRguXLSiH0rIpGBqTeHwrQdbkJTjr3QiIf1uP5t7ut, d: 08:36:24, a: 686, b: 686, c: ZeaJnBUa, }"] [2025/04/06 08:25:43.864 +08:00] [WARN] [diff.go:940] ["target had superfluous data"] [row="{ a: 687, b: 687, c: o4, ahRhZ: TRDQxX, d: 08:36:25, }"] "is-owner": true, run task successfully go: downloading github.com/jinzhu/now v1.1.5 go: downloading github.com/jinzhu/inflection v1.0.0 go: downloading github.com/siddontang/go-log v0.0.0-20180807004314-8d05993dda07 go: downloading github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726 go: downloading github.com/opencontainers/runtime-spec v1.0.2 go: downloading github.com/cilium/ebpf v0.4.0 go: downloading github.com/godbus/dbus/v5 v5.0.4 go: downloading github.com/sirupsen/logrus v1.9.3 go: downloading github.com/gin-contrib/sse v0.1.0 go: downloading github.com/ugorji/go/codec v1.2.11 go: downloading github.com/go-playground/validator/v10 v10.14.0 go: downloading github.com/pelletier/go-toml/v2 v2.0.8 go: downloading github.com/aws/smithy-go v1.13.5 [2025/04/06 08:25:43.964 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ ahRhZ: VgauzvAfkJ, d: 08:36:26, a: 688, b: 688, c: o, }"] [row2="{ a: 688, b: 688, c: 7DVPAB, ahRhZ: agJcVvjrHymScaYobB4WrNo5L6G, d: 08:36:26, }"] [2025/04/06 08:25:44.064 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 689, b: 689, c: o, ahRhZ: VgauzvAfkJ, d: 08:36:27, }"] [row2="{ c: hYoun6W, ahRhZ: kVxjceosFyUUuNg5GWBnH1EN, d: 08:36:27, a: 689, b: 689, }"] go: downloading github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 go: downloading github.com/eapache/go-resiliency v1.4.0 go: downloading github.com/eapache/queue v1.1.0 go: downloading github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 go: downloading github.com/hashicorp/go-multierror v1.1.1 go: downloading github.com/jcmturner/gofork v1.7.6 go: downloading github.com/jcmturner/gokrb5/v8 v8.4.4 go: downloading github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 go: downloading github.com/AthenZ/athenz v1.10.39 go: downloading golang.org/x/mod v0.22.0 go: downloading github.com/99designs/keyring v1.2.1 go: downloading github.com/linkedin/goavro/v2 v2.11.1 go: downloading github.com/spaolacci/murmur3 v1.1.0 go: downloading github.com/go-playground/universal-translator v0.18.1 go: downloading github.com/gabriel-vasile/mimetype v1.4.2 go: downloading github.com/leodido/go-urn v1.2.4 go: downloading github.com/hashicorp/errwrap v1.0.0 go: downloading github.com/dvsekhvalnov/jose2go v1.5.0 go: downloading github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c go: downloading github.com/mtibben/percent v0.2.1 [2025/04/06 08:25:44.165 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ c: o, ahRhZ: VgauzvAfkJ, d: 08:36:29, a: 690, b: 690, }"] [row2="{ a: 690, b: 690, c: 7NJVoi1a, ahRhZ: 2kApifxRMzwt, d: 08:36:29, }"] [2025/04/06 08:25:44.265 +08:00] [WARN] [diff.go:937] ["find different row"] [column=ahRhZ] [row1="{ a: 691, b: 691, c: o, ahRhZ: VgauzvAfkJ, d: 08:36:30, }"] [row2="{ ahRhZ: URzkinRJQCAE, d: 08:36:30, a: 691, b: 691, c: dTkoTfzAH, }"] [2025/04/06 08:25:44.365 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ b: 692, c: o, ahRhZ: VgauzvAfkJ, d: 08:36:31, a: 692, }"] [row2="{ b: 692, c: MNg, ahRhZ: hWIhw62675HqX6bY66ek1cjlxCN7, d: 08:36:31, a: 692, }"] go: downloading github.com/go-playground/locales v0.14.1 go: downloading github.com/jcmturner/aescts/v2 v2.0.0 go: downloading github.com/hashicorp/go-uuid v1.0.3 go: downloading github.com/jcmturner/dnsutils/v2 v2.0.0 go: downloading github.com/jcmturner/rpc/v2 v2.0.3 go: downloading github.com/soheilhy/cmux v0.1.5 go: downloading go.etcd.io/bbolt v1.3.9 go: downloading go.etcd.io/etcd/pkg/v3 v3.5.12 go: downloading github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75 go: downloading sigs.k8s.io/yaml v1.4.0 go: downloading github.com/xiang90/probing v0.0.0-20221125231312-a49e3df8f510 go: downloading github.com/jonboulle/clockwork v0.4.0 go: downloading go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.22.0 go: downloading github.com/grpc-ecosystem/grpc-gateway v1.16.0 go: downloading go.opentelemetry.io/otel/sdk v1.24.0 go: downloading go.etcd.io/etcd/raft/v3 v3.5.12 go: downloading go.etcd.io/etcd/client/v2 v2.305.12 go: downloading github.com/golang-jwt/jwt/v4 v4.5.1 [2025/04/06 08:25:44.465 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 693, b: 693, c: o, ahRhZ: VgauzvAfkJ, d: 08:36:32, }"] [row2="{ a: 693, b: 693, c: 1zpSxk, ahRhZ: ErxQ5CDWYuT2t, d: 08:36:32, }"] [2025/04/06 08:25:44.565 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 694, b: 694, c: o, ahRhZ: VgauzvAfkJ, d: 08:36:33, }"] [row2="{ a: 694, b: 694, c: 1B, ahRhZ: rJ9OJhc, d: 08:36:33, }"] [2025/04/06 08:25:44.666 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 695, b: 695, c: o, ahRhZ: VgauzvAfkJ, d: 08:36:34, }"] [row2="{ b: 695, c: niPnlSoYZ4, ahRhZ: ZpmVbTfx3EJiyEpIiIaxBLW4n6dlBNtHKH6FwHnx0q, d: 08:36:34, a: 695, }"] + set +x + tso='457152696924504065 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 457152696924504065 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + start_ts=457152696924504065 + run_cdc_server --workdir /tmp/tidb_cdc_test/sql_mode --binary cdc.test [Sun Apr 6 08:25:44 CST 2025] <<<<<< START cdc server in sql_mode case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sql_mode.1454314545.out server --log-file /tmp/tidb_cdc_test/sql_mode/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/sql_mode/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 go: downloading github.com/gorilla/websocket v1.5.1 go: downloading go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.22.0 go: downloading go.opentelemetry.io/proto/otlp v1.1.0 go: downloading github.com/cenkalti/backoff/v4 v4.2.1 go: downloading github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1 [2025/04/06 08:25:44.766 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ d: 08:36:35, a: 696, b: 696, c: o, ahRhZ: VgauzvAfkJ, }"] [row2="{ a: 696, b: 696, c: HgK, ahRhZ: EU2hJRovZzzLQtufisBHMGzbNZe7AbkhuK, d: 08:36:35, }"] [2025/04/06 08:25:44.866 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ c: o, ahRhZ: VgauzvAfkJ, d: 08:36:36, a: 697, b: 697, }"] [row2="{ a: 697, b: 697, c: LTFRwLu, ahRhZ: 18zNlqUNY6NKWI, d: 08:36:36, }"] [Sun Apr 6 08:25:44 CST 2025] <<<<<< START cdc server in multi_cdc_cluster case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_cdc_cluster.1240912411.out server --log-file /tmp/tidb_cdc_test/multi_cdc_cluster/cdcmult_cdc.server1.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_cdc_cluster/cdc_datamult_cdc.server1 --cluster-id test1 --addr 127.0.0.1:8300 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2025/04/06 08:25:44.966 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ c: o, ahRhZ: VgauzvAfkJ, d: 08:36:37, a: 698, b: 698, }"] [row2="{ a: 698, b: 698, c: NF, ahRhZ: 95nKaDANdwCwy5yXEdrsBudfUUb, d: 08:36:37, }"] [2025/04/06 08:25:45.066 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ ahRhZ: VgauzvAfkJ, d: 08:36:38, a: 699, b: 699, c: o, }"] [row2="{ b: 699, c: 3GpQF, ahRhZ: l3GIrSJMC5dUOes4ij6Wj, d: 08:36:38, a: 699, }"] [2025/04/06 08:25:45.167 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 700, b: 700, c: o, ahRhZ: VgauzvAfkJ, d: 08:36:39, }"] [row2="{ a: 700, b: 700, c: q, ahRhZ: qkyy9rKx0qwPu5e0mrsaLeqvkptdwRGa, d: 08:36:39, }"] [2025/04/06 08:25:45.267 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ ahRhZ: VgauzvAfkJ, d: 08:36:40, a: 701, b: 701, c: o, }"] [row2="{ a: 701, b: 701, c: 1FAkH, ahRhZ: yiCrDKJ, d: 08:36:40, }"] [2025/04/06 08:25:45.367 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ b: 702, c: o, ahRhZ: VgauzvAfkJ, d: 08:37:49, a: 702, }"] [row2="{ a: 702, b: 702, c: akIcZK4, ahRhZ: wrEAjgZcOe67w9BnGGz81y3vIbRGhS6uxEf, d: 08:37:49, }"] [2025/04/06 08:25:45.467 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ d: 08:36:42, a: 703, b: 703, c: o, ahRhZ: VgauzvAfkJ, }"] [row2="{ a: 703, b: 703, c: oxGm, ahRhZ: FSyB3fTjz2Nvfa5ioj, d: 08:36:42, }"] [2025/04/06 08:25:45.567 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 704, b: 704, c: o, ahRhZ: VgauzvAfkJ, d: 08:36:43, }"] [row2="{ a: 704, b: 704, c: yFPAH3c, ahRhZ: BR2Pvoi25c7gpM9evVMgTAyQz9Y83K6, d: 08:36:43, }"] [2025/04/06 08:25:45.668 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 705, b: 705, c: o, ahRhZ: VgauzvAfkJ, d: 08:36:44, }"] [row2="{ d: 08:36:44, a: 705, b: 705, c: d, ahRhZ: LPmwxYTKIKw1, }"] owner pid: 6686 owner id b2fc3695-c374-49af-865d-796d9a81d5d6 [2025/04/06 08:25:45.768 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 706, b: 706, c: o, ahRhZ: VgauzvAfkJ, d: 08:36:45, }"] [row2="{ ahRhZ: nUHVrNHW7YWPTrwS73HNv1SOmbX6WydOEIgD81Ie0, d: 08:36:45, a: 706, b: 706, c: Zhyn4, }"] [2025/04/06 08:25:45.868 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 707, b: 707, c: o, ahRhZ: VgauzvAfkJ, d: 08:36:46, }"] [row2="{ c: WHkROw1, ahRhZ: pUFp3vYq2N6RkKnSVMtRwGB3zTc2rFSlQKkAMRBo73MDi, d: 08:36:46, a: 707, b: 707, }"] [2025/04/06 08:25:45.968 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 709, b: 709, c: o, ahRhZ: VgauzvAfkJ, d: 08:36:48, }"] [row2="{ b: 709, c: 1gw, ahRhZ: LV69WelFoyacnRGls7c4, d: 08:36:48, a: 709, }"] [2025/04/06 08:25:46.068 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ c: o, ahRhZ: VgauzvAfkJ, d: 08:36:49, a: 710, b: 710, }"] [row2="{ a: 710, b: 710, c: SMzKCx, ahRhZ: Tf2mbrx0ZTg4YH, d: 08:36:49, }"] [2025/04/06 08:25:46.169 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 711, b: 711, c: o, ahRhZ: VgauzvAfkJ, d: 08:36:50, }"] [row2="{ a: 711, b: 711, c: 5F0f6XK, ahRhZ: 6DaSQvKP3S3F37ieqziF4mVGpMJh2BRIXRr375u6M, d: 08:36:50, }"] [2025/04/06 08:25:46.269 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ d: 08:36:51, a: 712, b: 712, c: o, ahRhZ: VgauzvAfkJ, }"] [row2="{ a: 712, b: 712, c: A, ahRhZ: xuMcPn9M, d: 08:36:51, }"] [2025/04/06 08:25:46.369 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ c: o, ahRhZ: VgauzvAfkJ, d: 08:36:52, a: 713, b: 713, }"] [row2="{ ahRhZ: RvgFrZbELRghaZVCZWfvlGDOlfFvcTwsY6m5s8g, d: 08:36:52, a: 713, b: 713, c: NGw0, }"] [2025/04/06 08:25:46.469 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 714, b: 714, c: o, ahRhZ: VgauzvAfkJ, d: 08:36:53, }"] [row2="{ a: 714, b: 714, c: p6Xh, ahRhZ: UJiYaYFpuYbDyjaCLYPzOEF79, d: 08:36:53, }"] [2025/04/06 08:25:46.569 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 715, b: 715, c: o, ahRhZ: VgauzvAfkJ, d: 08:36:54, }"] [row2="{ a: 715, b: 715, c: A, ahRhZ: gI0DBeS3rREqlfi, d: 08:36:54, }"] [2025/04/06 08:25:46.670 +08:00] [WARN] [diff.go:937] ["find different row"] [column=ahRhZ] [row1="{ b: 717, c: o, ahRhZ: VgauzvAfkJ, d: 08:36:56, a: 717, }"] [row2="{ a: 717, b: 717, c: 9Fjsm, ahRhZ: gwO6, d: 08:36:56, }"] [2025/04/06 08:25:46.770 +08:00] [WARN] [diff.go:937] ["find different row"] [column=ahRhZ] [row1="{ a: 718, b: 718, c: o, ahRhZ: VgauzvAfkJ, d: 08:36:57, }"] [row2="{ b: 718, c: ZjGzJm, ahRhZ: WYTNSfuYL, d: 08:36:57, a: 718, }"] [2025/04/06 08:25:46.870 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ b: 719, c: o, ahRhZ: VgauzvAfkJ, d: 08:36:58, a: 719, }"] [row2="{ b: 719, c: rqjRRYRV, ahRhZ: kjsqMR7yPpkHkKYLX, d: 08:36:58, a: 719, }"] =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/owner_remove_table_error/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [2025/04/06 08:25:46.970 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 720, b: 720, c: o, ahRhZ: VgauzvAfkJ, d: 08:36:59, }"] [2025/04/06 08:25:47.071 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ c: o, ahRhZ: VgauzvAfkJ, d: 08:37:00, a: 721, b: 721, }"] [2025/04/06 08:25:47.171 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ c: o, ahRhZ: VgauzvAfkJ, d: 08:37:01, a: 722, b: 722, }"] changefeed id: e684662d-022f-4a79-9ef3-0a59dba96193 curl -s -d "cf-id=e684662d-022f-4a79-9ef3-0a59dba96193&admin-job=1" http://127.0.0.1:8300/capture/owner/admin | grep true "status": true, run task successfully % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 89 100 35 100 54 499 770 --:--:-- --:--:-- --:--:-- 782 { "status": true, "message": "" }nonempty select id, val from test.availability1 where id=4 and val=4 TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 1-th time, retry later [2025/04/06 08:25:47.271 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 723, b: 723, c: o, ahRhZ: VgauzvAfkJ, d: 08:37:02, }"] [2025/04/06 08:25:47.371 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ b: 724, c: o, ahRhZ: VgauzvAfkJ, d: 08:37:03, a: 724, }"] check_redo_resolved_ts cb6b1b7f-ab29-4852-a097-f142c1ff4ff0 457152696404934662 file:///tmp/tidb_cdc_test/consistent_replicate_storage_file/redo /tmp/tidb_cdc_test/consistent_replicate_storage_file/cdc_data/redo/cb6b1b7f-ab29-4852-a097-f142c1ff4ff0/meta run task successfully [2025/04/06 08:25:47.472 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 725, b: 725, c: o, ahRhZ: VgauzvAfkJ, d: 08:37:04, }"] [2025/04/06 08:25:47.572 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 726, b: 726, c: o, ahRhZ: VgauzvAfkJ, d: 08:37:05, }"] [2025/04/06 08:25:47.672 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ ahRhZ: VgauzvAfkJ, d: 08:37:06, a: 727, b: 727, c: o, }"] =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/ci_collation_compatibility/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 06 Apr 2025 00:25:47 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/test-1 {UpstreamID:7489989357300557951 Namespace:default ID:test-1 SinkURI:mysql://root@127.0.0.1:3306/?max-txn-row=1 CreateTime:2025-04-06 08:24:18.991219273 +0800 CST StartTs:457152673551482881 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003f494d0 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457152674914369541} {CheckpointTs:457152678558171144 MinTableBarrierTs:457152678558171144 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/191377ed-3ec1-4853-8612-4ab86035b8bf {"id":"191377ed-3ec1-4853-8612-4ab86035b8bf","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899144} /tidb/cdc/default/__cdc_meta__/capture/623ac603-413a-4a2c-99ce-f293ec175a86 {"id":"623ac603-413a-4a2c-99ce-f293ec175a86","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899055} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087ba13feb 623ac603-413a-4a2c-99ce-f293ec175a86 /tidb/cdc/default/__cdc_meta__/owner/223196087cfe81dd 191377ed-3ec1-4853-8612-4ab86035b8bf /tidb/cdc/default/default/changefeed/info/test-1 {"upstream-id":7489989357300557951,"namespace":"default","changefeed-id":"test-1","sink-uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create-time":"2025-04-06T08:24:18.991219273+08:00","start-ts":457152673551482881,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457152674914369541} /tidb/cdc/default/default/changefeed/status/test-1 {"checkpoint-ts":457152678558171144,"min-table-barrier-ts":457152678558171144,"admin-job-type":0} /tidb/cdc/default/default/task/position/191377ed-3ec1-4853-8612-4ab86035b8bf/test-1 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/623ac603-413a-4a2c-99ce-f293ec175a86/test-1 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489989357300557951 {"id":7489989357300557951,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/test-1 {UpstreamID:7489989357300557951 Namespace:default ID:test-1 SinkURI:mysql://root@127.0.0.1:3306/?max-txn-row=1 CreateTime:2025-04-06 08:24:18.991219273 +0800 CST StartTs:457152673551482881 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003f494d0 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457152674914369541} {CheckpointTs:457152678558171144 MinTableBarrierTs:457152678558171144 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/191377ed-3ec1-4853-8612-4ab86035b8bf {"id":"191377ed-3ec1-4853-8612-4ab86035b8bf","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899144} /tidb/cdc/default/__cdc_meta__/capture/623ac603-413a-4a2c-99ce-f293ec175a86 {"id":"623ac603-413a-4a2c-99ce-f293ec175a86","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899055} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087ba13feb 623ac603-413a-4a2c-99ce-f293ec175a86 /tidb/cdc/default/__cdc_meta__/owner/223196087cfe81dd 191377ed-3ec1-4853-8612-4ab86035b8bf /tidb/cdc/default/default/changefeed/info/test-1 {"upstream-id":7489989357300557951,"namespace":"default","changefeed-id":"test-1","sink-uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create-time":"2025-04-06T08:24:18.991219273+08:00","start-ts":457152673551482881,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457152674914369541} /tidb/cdc/default/default/changefeed/status/test-1 {"checkpoint-ts":457152678558171144,"min-table-barrier-ts":457152678558171144,"admin-job-type":0} /tidb/cdc/default/default/task/position/191377ed-3ec1-4853-8612-4ab86035b8bf/test-1 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/623ac603-413a-4a2c-99ce-f293ec175a86/test-1 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489989357300557951 {"id":7489989357300557951,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: changefeedID: default/test-1 {UpstreamID:7489989357300557951 Namespace:default ID:test-1 SinkURI:mysql://root@127.0.0.1:3306/?max-txn-row=1 CreateTime:2025-04-06 08:24:18.991219273 +0800 CST StartTs:457152673551482881 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003f494d0 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457152674914369541} {CheckpointTs:457152678558171144 MinTableBarrierTs:457152678558171144 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/191377ed-3ec1-4853-8612-4ab86035b8bf {"id":"191377ed-3ec1-4853-8612-4ab86035b8bf","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899144} /tidb/cdc/default/__cdc_meta__/capture/623ac603-413a-4a2c-99ce-f293ec175a86 {"id":"623ac603-413a-4a2c-99ce-f293ec175a86","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899055} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087ba13feb 623ac603-413a-4a2c-99ce-f293ec175a86 /tidb/cdc/default/__cdc_meta__/owner/223196087cfe81dd 191377ed-3ec1-4853-8612-4ab86035b8bf /tidb/cdc/default/default/changefeed/info/test-1 {"upstream-id":7489989357300557951,"namespace":"default","changefeed-id":"test-1","sink-uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create-time":"2025-04-06T08:24:18.991219273+08:00","start-ts":457152673551482881,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457152674914369541} /tidb/cdc/default/default/changefeed/status/test-1 {"checkpoint-ts":457152678558171144,"min-table-barrier-ts":457152678558171144,"admin-job-type":0} /tidb/cdc/default/default/task/position/191377ed-3ec1-4853-8612-4ab86035b8bf/test-1 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/623ac603-413a-4a2c-99ce-f293ec175a86/test-1 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489989357300557951 {"id":7489989357300557951,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + SINK_URI='mysql://root@127.0.0.1:3306/?max-txn-row=1' + run_cdc_cli changefeed create --start-ts=457152696924504065 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-2 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sql_mode.cli.14599.out cli changefeed create --start-ts=457152696924504065 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-2 Create changefeed successfully! ID: test-2 Info: {"upstream_id":7489989357300557951,"namespace":"default","id":"test-2","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2025-04-06T08:25:47.834918885+08:00","start_ts":457152696924504065,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457152696924504065,"checkpoint_ts":457152696924504065,"checkpoint_time":"2025-04-06 08:25:42.931"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... [2025/04/06 08:25:47.772 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 728, b: 728, c: o, ahRhZ: VgauzvAfkJ, d: 08:37:07, }"] [2025/04/06 08:25:47.872 +08:00] [WARN] [diff.go:942] ["target lack data"] [row="{ a: 729, b: 729, c: o, ahRhZ: VgauzvAfkJ, d: 08:37:08, }"] [2025/04/06 08:25:47.972 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ b: 730, c: o, ahRhZ: VgauzvAfkJ, d: 08:37:09, a: 730, }"] [row2="{ b: 730, c: M9aTt6cBy, ahRhZ: 0XfiKoFj2zuoJvTGfty0FoKUmEhJaFcXGhVl4, d: 08:37:09, a: 730, }"] wait process cdc.test exit for 1-th time... [2025/04/06 08:25:48.073 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ ahRhZ: VgauzvAfkJ, d: 08:39:03, a: 731, b: 731, c: o, }"] [row2="{ a: 731, b: 731, c: qELJfne, ahRhZ: iGtSFeZpGaiH4cr88FdU5fkOaUmhQ5CAmZeYh9, d: 08:39:03, }"] [2025/04/06 08:25:48.173 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ c: o, ahRhZ: VgauzvAfkJ, d: 08:37:11, a: 732, b: 732, }"] [row2="{ b: 732, c: JOpa5F9, ahRhZ: 6ZzvihkEy8HbKgxWS, d: 08:37:11, a: 732, }"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 06 Apr 2025 00:25:47 GMT < Content-Length: 812 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/test1/__cdc_meta__/capture/6e635749-eea1-48cd-9c02-434ae894d61e {"id":"6e635749-eea1-48cd-9c02-434ae894d61e","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899145} /tidb/cdc/test1/__cdc_meta__/meta/meta-version 1 /tidb/cdc/test1/__cdc_meta__/owner/223196087cf8c5fb 6e635749-eea1-48cd-9c02-434ae894d61e /tidb/cdc/test1/default/upstream/7489989742764661549 {"id":7489989742764661549,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/test1/__cdc_meta__/capture/6e635749-eea1-48cd-9c02-434ae894d61e {"id":"6e635749-eea1-48cd-9c02-434ae894d61e","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899145} /tidb/cdc/test1/__cdc_meta__/meta/meta-version 1 /tidb/cdc/test1/__cdc_meta__/owner/223196087cf8c5fb 6e635749-eea1-48cd-9c02-434ae894d61e /tidb/cdc/test1/default/upstream/7489989742764661549 {"id":7489989742764661549,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/test1/__cdc_meta__/capture/6e635749-eea1-48cd-9c02-434ae894d61e {"id":"6e635749-eea1-48cd-9c02-434ae894d61e","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899145} /tidb/cdc/test1/__cdc_meta__/meta/meta-version 1 /tidb/cdc/test1/__cdc_meta__/owner/223196087cf8c5fb 6e635749-eea1-48cd-9c02-434ae894d61e /tidb/cdc/test1/default/upstream/7489989742764661549 {"id":7489989742764661549,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Sun Apr 6 08:25:47 CST 2025] <<<<<< START cdc server in multi_cdc_cluster case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_cdc_cluster.1245512457.out server --log-file /tmp/tidb_cdc_test/multi_cdc_cluster/cdcmult_cdc.server2.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_cdc_cluster/cdc_datamult_cdc.server2 --cluster-id test2 --addr 127.0.0.1:8301 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 wait process cdc.test exit for 2-th time... [2025/04/06 08:25:48.273 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ c: o, ahRhZ: VgauzvAfkJ, d: 08:37:12, a: 733, b: 733, }"] [row2="{ b: 733, c: IYFIiU5uM, ahRhZ: PZa9sDJL6R5SWq9Tm7kEY6Otr, d: 08:37:12, a: 733, }"] [2025/04/06 08:25:48.373 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 734, b: 734, c: o, ahRhZ: VgauzvAfkJ, d: 08:37:13, }"] [row2="{ b: 734, c: PQcLwHUKG, ahRhZ: P0BogRNWIEOTDIM, d: 08:37:13, a: 734, }"] [2025/04/06 08:25:48.474 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ c: o, ahRhZ: VgauzvAfkJ, d: 08:37:14, a: 735, b: 735, }"] [row2="{ a: 735, b: 735, c: GbUoczfi, ahRhZ: 7yamRj1wFMaAx74CwfuJP92NZHTFKr3xA, d: 08:37:14, }"] [2025/04/06 08:25:48.574 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ d: 08:37:15, a: 736, b: 736, c: o, ahRhZ: VgauzvAfkJ, }"] [row2="{ a: 736, b: 736, c: zwf2ctCE, ahRhZ: 8KnGLq6FD7AH61F8EIbLRQSUKDp, d: 08:37:15, }"] [2025/04/06 08:25:48.674 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ d: 08:37:16, a: 737, b: 737, c: o, ahRhZ: VgauzvAfkJ, }"] [row2="{ ahRhZ: ksxwrKn, d: 08:37:16, a: 737, b: 737, c: KaNbySJn, }"] wait process cdc.test exit for 3-th time... [2025/04/06 08:25:48.774 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ ahRhZ: VgauzvAfkJ, d: 08:37:17, a: 738, b: 738, c: o, }"] [row2="{ ahRhZ: LosBewvafTJ, d: 08:37:17, a: 738, b: 738, c: Cgutt, }"] [2025/04/06 08:25:48.874 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ c: o, ahRhZ: VgauzvAfkJ, d: 08:37:18, a: 739, b: 739, }"] [row2="{ a: 739, b: 739, c: ITx0f8O86, ahRhZ: QB8xXTxAa1QGPo4mqqn8nw, d: 08:37:18, }"] [2025/04/06 08:25:48.975 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 740, b: 740, c: o, ahRhZ: VgauzvAfkJ, d: 08:37:19, }"] [row2="{ a: 740, b: 740, c: RFSV, ahRhZ: 38xx5B526QXGZsB31Qtr006cT, d: 08:37:19, }"] go: downloading github.com/ardielle/ardielle-go v1.5.2 nonempty select id, val from test.availability1 where id=4 and val=4 [2025/04/06 08:25:49.075 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 741, b: 741, c: o, ahRhZ: VgauzvAfkJ, d: 08:37:20, }"] [row2="{ d: 08:37:20, a: 741, b: 741, c: EH8eIAMk, ahRhZ: K5EcjBy4sbJONol3vqNsh, }"] [2025/04/06 08:25:49.175 +08:00] [WARN] [diff.go:937] ["find different row"] [column=ahRhZ] [row1="{ a: 742, b: 742, c: o, ahRhZ: VgauzvAfkJ, d: 08:37:21, }"] [row2="{ d: 08:37:21, a: 742, b: 742, c: Pj9o, ahRhZ: lExeF9uKZeKyoE, }"] TEST FAILED: OUTPUT DOES NOT CONTAIN 'id:' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ run task failed 2-th time, retry later \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } [2025/04/06 08:25:49.275 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 743, b: 743, c: o, ahRhZ: VgauzvAfkJ, d: 08:37:22, }"] [row2="{ a: 743, b: 743, c: rvm, ahRhZ: 22Aq85v9MSmp83UcDAVoNxp2Q9UQ9Tu94LlxyxpNWJ3F, d: 08:37:22, }"] [2025/04/06 08:25:49.375 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 744, b: 744, c: o, ahRhZ: VgauzvAfkJ, d: 08:37:23, }"] [row2="{ a: 744, b: 744, c: jFaqcsa, ahRhZ: QIDra96EpsBmd, d: 08:37:23, }"] [2025/04/06 08:25:49.476 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 745, b: 745, c: o, ahRhZ: VgauzvAfkJ, d: 08:37:24, }"] [row2="{ a: 745, b: 745, c: MGeuFLkKCs, ahRhZ: UgoUYKsryUcbR8x1LG, d: 08:37:24, }"] cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [2025/04/06 08:25:49.471 +08:00] [INFO] [helper.go:54] ["init log"] [file=] [level=info] [2025/04/06 08:25:49.472 +08:00] [INFO] [apply.go:78] ["set memory limit"] [memoryLimit=10737418240] [2025/04/06 08:25:49.473 +08:00] [INFO] [apply.go:94] ["Start http pprof server"] [addr=127.0.0.1:6060] [2025/04/06 08:25:49.473 +08:00] [INFO] [mem_quota.go:82] ["New memory quota"] [namespace=default] [changefeed=redo-applier] [total=1073741824] [2025/04/06 08:25:49.473 +08:00] [INFO] [redo.go:167] ["apply redo log starts"] [checkpointTs=457152690834374661] [resolvedTs=457152697741869057] [2025/04/06 08:25:49.475 +08:00] [INFO] [tz.go:35] ["Use the timezone of the TiCDC server machine"] [timezoneName=System] [timezone=Asia/Shanghai] [2025/04/06 08:25:49.475 +08:00] [WARN] [config.go:428] ["Because time-zone is not specified, the timezone of the TiCDC server will be used. We recommend that you specify the time-zone explicitly. Please make sure that the timezone of the TiCDC server, sink-uri and the downstream database are consistent. If the downstream database does not load the timezone information, you can refer to https://dev.mysql.com/doc/refman/8.0/en/mysql-tzinfo-to-sql.html."] [timezone=Asia/Shanghai] [2025/04/06 08:25:49.475 +08:00] [ERROR] [config.go:216] ["The TiDB source ID should never be set to 0. Please report it as a bug. The default value will be used: 1."] [tidbSourceID=0] [2025/04/06 08:25:49.476 +08:00] [INFO] [file.go:114] ["succeed to download and sort redo logs"] [type=ddl] [duration=3.602119ms] [2025/04/06 08:25:49.482 +08:00] [INFO] [db_helper.go:161] ["sink uri is configured"] [dsn="normal:******@tcp(127.0.0.1:3306)/?interpolateParams=true&multiStatements=true&allow_auto_random_explicit_insert=1&charset=utf8mb4&foreign_key_checks=0&maxAllowedPacket=0&readTimeout=2m&sql_mode=%22NO_ENGINE_SUBSTITUTION%2CIGNORE_SPACE%2CALLOW_INVALID_DATES%2CONLY_FULL_GROUP_BY%2CNO_AUTO_VALUE_ON_ZERO%22&tidb_enable_external_ts_read=%22OFF%22&tidb_placement_mode=%22ignore%22&tidb_txn_mode=optimistic&time_zone=%22Asia%2FShanghai%22&timeout=2m&transaction_isolation=%22READ-COMMITTED%22&writeTimeout=2m"] [2025/04/06 08:25:49.485 +08:00] [INFO] [mysql.go:195] ["MySQL backends is created"] [changefeed=.] [workerCount=16] [forceReplicate=false] [2025/04/06 08:25:49.486 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/06 08:25:49.486 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/06 08:25:49.486 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/06 08:25:49.486 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/06 08:25:49.486 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/06 08:25:49.486 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/06 08:25:49.486 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/06 08:25:49.486 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/06 08:25:49.486 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/06 08:25:49.486 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/06 08:25:49.486 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/06 08:25:49.486 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/06 08:25:49.486 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/06 08:25:49.486 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/06 08:25:49.486 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/06 08:25:49.486 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/06 08:25:49.486 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=1] [2025/04/06 08:25:49.486 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=3] [2025/04/06 08:25:49.486 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=2] [2025/04/06 08:25:49.486 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=0] [2025/04/06 08:25:49.486 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=5] [2025/04/06 08:25:49.486 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=6] [2025/04/06 08:25:49.486 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=4] [2025/04/06 08:25:49.487 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=9] [2025/04/06 08:25:49.487 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=7] [2025/04/06 08:25:49.487 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=8] [2025/04/06 08:25:49.487 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=11] [2025/04/06 08:25:49.487 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=12] [2025/04/06 08:25:49.487 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=14] [2025/04/06 08:25:49.487 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=13] [2025/04/06 08:25:49.487 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=15] [2025/04/06 08:25:49.487 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=10] [2025/04/06 08:25:49.487 +08:00] [INFO] [tz.go:35] ["Use the timezone of the TiCDC server machine"] [timezoneName=System] [timezone=Asia/Shanghai] [2025/04/06 08:25:49.487 +08:00] [WARN] [config.go:428] ["Because time-zone is not specified, the timezone of the TiCDC server will be used. We recommend that you specify the time-zone explicitly. Please make sure that the timezone of the TiCDC server, sink-uri and the downstream database are consistent. If the downstream database does not load the timezone information, you can refer to https://dev.mysql.com/doc/refman/8.0/en/mysql-tzinfo-to-sql.html."] [timezone=Asia/Shanghai] [2025/04/06 08:25:49.487 +08:00] [ERROR] [config.go:216] ["The TiDB source ID should never be set to 0. Please report it as a bug. The default value will be used: 1."] [tidbSourceID=0] [2025/04/06 08:25:49.491 +08:00] [INFO] [db_helper.go:161] ["sink uri is configured"] [dsn="normal:******@tcp(127.0.0.1:3306)/?interpolateParams=true&multiStatements=true&allow_auto_random_explicit_insert=1&charset=utf8mb4&foreign_key_checks=0&maxAllowedPacket=0&readTimeout=2m&sql_mode=%22NO_AUTO_VALUE_ON_ZERO%2CONLY_FULL_GROUP_BY%2CNO_ENGINE_SUBSTITUTION%2CIGNORE_SPACE%2CALLOW_INVALID_DATES%22&tidb_enable_external_ts_read=%22OFF%22&tidb_placement_mode=%22ignore%22&tidb_txn_mode=optimistic&time_zone=%22Asia%2FShanghai%22&timeout=2m&transaction_isolation=%22READ-COMMITTED%22&writeTimeout=2m"] [2025/04/06 08:25:49.494 +08:00] [WARN] [mysql_ddl_sink.go:291] ["please set `has-vector-type` to be true if a column is vector type when the downstream is not TiDB or TiDB version less than specify version"] [hasVectorType=false] [supportVectorVersion=8.4.0] [2025/04/06 08:25:49.494 +08:00] [INFO] [mysql_ddl_sink.go:122] ["MySQL DDL sink is created"] [namespace=] [changefeed=] [Pipeline] // timeout [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-periodics_cdc_integration_mysql_test-234/tiflow-cdc already exists) [2025/04/06 08:25:49.576 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ d: 08:37:25, a: 746, b: 746, c: o, ahRhZ: VgauzvAfkJ, }"] [row2="{ a: 746, b: 746, c: 6Rx5, ahRhZ: 1qylyRRNJHpoNh4WxnS8osq9SgBUNTe0CVYuGY6X0f, d: 08:37:25, }"] [2025/04/06 08:25:49.676 +08:00] [WARN] [diff.go:937] ["find different row"] [column=ahRhZ] [row1="{ a: 747, b: 747, c: o, ahRhZ: VgauzvAfkJ, d: 08:37:26, }"] [row2="{ ahRhZ: J4UELoH7c, d: 08:37:26, a: 747, b: 747, c: qVtjthqkxz, }"] [Pipeline] // cache [Pipeline] } + set +x + run_sql 'use test; create table t2(id bigint primary key, a date); insert into t2 values(1, '\''2023-02-08'\'');' 127.0.0.1 4000 [Pipeline] // dir [Pipeline] } [2025/04/06 08:25:49.776 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 748, b: 748, c: o, ahRhZ: VgauzvAfkJ, d: 08:37:27, }"] [row2="{ a: 748, b: 748, c: 1hL, ahRhZ: EYLId2RweGgL3Lb4NnXHl6vC2VNfZSlS, d: 08:37:27, }"] [2025/04/06 08:25:49.876 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 749, b: 749, c: o, ahRhZ: VgauzvAfkJ, d: 08:37:28, }"] [row2="{ a: 749, b: 749, c: uO, ahRhZ: SEtLsdODYuQBchSXI4N4, d: 08:37:28, }"] [2025/04/06 08:25:49.977 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ c: o, ahRhZ: VgauzvAfkJ, d: 08:37:29, a: 750, b: 750, }"] [row2="{ d: 08:37:29, a: 750, b: 750, c: WN2c1, ahRhZ: edcQdGMltB76p4Nn5I5OoRa8CXXX4zBfO3tvdcgI, }"] [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [2025/04/06 08:25:50.077 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 751, b: 751, c: o, ahRhZ: VgauzvAfkJ, d: 08:37:30, }"] [row2="{ a: 751, b: 751, c: XFt8A, ahRhZ: IynQW6SizQrgVL7L16l, d: 08:37:30, }"] [2025/04/06 08:25:50.177 +08:00] [WARN] [diff.go:937] ["find different row"] [column=ahRhZ] [row1="{ a: 752, b: 752, c: o, ahRhZ: VgauzvAfkJ, d: 08:37:31, }"] [row2="{ a: 752, b: 752, c: j, ahRhZ: I3Q9bEcBYnruDoTHFL3ijQbxOTbLilOHNru3LZL8d6Na, d: 08:37:31, }"] [Pipeline] // withEnv start tidb cluster in /tmp/tidb_cdc_test/owner_remove_table_error Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... [Pipeline] } [Pipeline] // node [Pipeline] } [2025/04/06 08:25:50.277 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ c: o, ahRhZ: VgauzvAfkJ, d: 08:37:32, a: 753, b: 753, }"] [row2="{ b: 753, c: 1, ahRhZ: 6ZdajvH3bSg, d: 08:37:32, a: 753, }"] [2025/04/06 08:25:50.377 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 754, b: 754, c: o, ahRhZ: VgauzvAfkJ, d: 08:38:52, }"] [row2="{ b: 754, c: rRnMQXWTS, ahRhZ: bbURM5mfUDTpKSAClLYRZWwS6VgeaqUQcuOIyD9vUuEh, d: 08:38:52, a: 754, }"] [2025/04/06 08:25:50.478 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ b: 755, c: o, ahRhZ: VgauzvAfkJ, d: 08:37:34, a: 755, }"] [row2="{ a: 755, b: 755, c: AtLzFOC0d2, ahRhZ: 4c6UIuJMb8FnqZPzMPKTXjjR6EeT9lLS, d: 08:37:34, }"] [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [2025/04/06 08:25:50.578 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 756, b: 756, c: o, ahRhZ: VgauzvAfkJ, d: 08:37:35, }"] [row2="{ a: 756, b: 756, c: wQKiko, ahRhZ: GLlOMC5kxGFRnN62TxgH, d: 08:37:35, }"] [2025/04/06 08:25:50.678 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 757, b: 757, c: o, ahRhZ: VgauzvAfkJ, d: 08:37:36, }"] [row2="{ a: 757, b: 757, c: AWK7Dsx, ahRhZ: vcm0iUrfQ, d: 08:37:36, }"] [Pipeline] } + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > [2025/04/06 08:25:50.778 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 758, b: 758, c: o, ahRhZ: VgauzvAfkJ, d: 08:37:37, }"] [row2="{ a: 758, b: 758, c: jJRvDj9, ahRhZ: 69PEgbDaKc6VNE7hRYg4eqm4fuWiUuMli, d: 08:37:37, }"] [2025/04/06 08:25:50.878 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 759, b: 759, c: o, ahRhZ: VgauzvAfkJ, d: 08:37:38, }"] [row2="{ ahRhZ: FFcVnwVq4wcujYwPaB, d: 08:37:38, a: 759, b: 759, c: 36oMME, }"] [2025/04/06 08:25:50.979 +08:00] [WARN] [diff.go:937] ["find different row"] [column=ahRhZ] [row1="{ a: 760, b: 760, c: o, ahRhZ: VgauzvAfkJ, d: 08:37:39, }"] [row2="{ a: 760, b: 760, c: OA, ahRhZ: EInXE, d: 08:37:39, }"] start tidb cluster in /tmp/tidb_cdc_test/ci_collation_compatibility Starting Upstream PD... < HTTP/1.1 200 OK < Date: Sun, 06 Apr 2025 00:25:50 GMT < Content-Length: 812 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/test2/__cdc_meta__/capture/c4f58222-cbd5-4f7d-8619-be4240a9ae6b {"id":"c4f58222-cbd5-4f7d-8619-be4240a9ae6b","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899148} /tidb/cdc/test2/__cdc_meta__/meta/meta-version 1 /tidb/cdc/test2/__cdc_meta__/owner/223196087cf8c626 c4f58222-cbd5-4f7d-8619-be4240a9ae6b /tidb/cdc/test2/default/upstream/7489989742764661549 {"id":7489989742764661549,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/test2/__cdc_meta__/capture/c4f58222-cbd5-4f7d-8619-be4240a9ae6b {"id":"c4f58222-cbd5-4f7d-8619-be4240a9ae6b","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899148} /tidb/cdc/test2/__cdc_meta__/meta/meta-version 1 /tidb/cdc/test2/__cdc_meta__/owner/223196087cf8c626 c4f58222-cbd5-4f7d-8619-be4240a9ae6b /tidb/cdc/test2/default/upstream/7489989742764661549 {"id":7489989742764661549,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/test2/__cdc_meta__/capture/c4f58222-cbd5-4f7d-8619-be4240a9ae6b {"id":"c4f58222-cbd5-4f7d-8619-be4240a9ae6b","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899148} /tidb/cdc/test2/__cdc_meta__/meta/meta-version 1 /tidb/cdc/test2/__cdc_meta__/owner/223196087cf8c626 c4f58222-cbd5-4f7d-8619-be4240a9ae6b /tidb/cdc/test2/default/upstream/7489989742764661549 {"id":7489989742764661549,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_cdc_cluster.cli.12514.out cli changefeed create --start-ts=457152696683593731 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ --server http://127.0.0.1:8300 --config=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/multi_cdc_cluster/conf/changefeed1.toml =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/savepoint/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... [2025/04/06 08:25:51.079 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 761, b: 761, c: o, ahRhZ: VgauzvAfkJ, d: 08:37:40, }"] [row2="{ b: 761, c: YJ38tlwBE, ahRhZ: YwfVmoIdlG7wg2JoKocrmq0w5b7j3h40dOkFaqu, d: 08:37:40, a: 761, }"] [2025/04/06 08:25:51.179 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ ahRhZ: VgauzvAfkJ, d: 08:37:41, a: 762, b: 762, c: o, }"] [row2="{ b: 762, c: J, ahRhZ: rzx26SVcK2EByXfiyd70uwdXbpJB6sThdiFh9B, d: 08:37:41, a: 762, }"] [2025/04/06 08:25:51.279 +08:00] [WARN] [diff.go:937] ["find different row"] [column=ahRhZ] [row1="{ a: 763, b: 763, c: o, ahRhZ: VgauzvAfkJ, d: 08:37:42, }"] [row2="{ d: 08:37:42, a: 763, b: 763, c: 6aNLQ, ahRhZ: jMz2Cuj8mm30JLTmU4, }"] [2025/04/06 08:25:51.380 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 764, b: 764, c: o, ahRhZ: VgauzvAfkJ, d: 08:37:43, }"] [row2="{ ahRhZ: VtNCso026yBVKfGutcfjUxnWzWzC, d: 08:37:43, a: 764, b: 764, c: Gva9aq97f8, }"] [2025/04/06 08:25:51.480 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ c: o, ahRhZ: VgauzvAfkJ, d: 08:37:44, a: 765, b: 765, }"] [row2="{ a: 765, b: 765, c: yu6VG0, ahRhZ: FbiBdbe9Ssf0Ath8pHqjWomJWI, d: 08:37:44, }"] Create changefeed successfully! ID: bead60a6-052b-4294-8bb2-8970dd5054d9 Info: {"upstream_id":7489989742764661549,"namespace":"default","id":"bead60a6-052b-4294-8bb2-8970dd5054d9","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-06T08:25:51.338193344+08:00","start_ts":457152696683593731,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["test.multi_cdc1"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457152696683593731,"checkpoint_ts":457152696683593731,"checkpoint_time":"2025-04-06 08:25:42.012"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... [2025/04/06 08:25:51.580 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 766, b: 766, c: o, ahRhZ: VgauzvAfkJ, d: 08:37:45, }"] [row2="{ a: 766, b: 766, c: 9xqrRw, ahRhZ: IY7aaHRixa5j2M, d: 08:37:45, }"] [2025/04/06 08:25:51.680 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ c: o, ahRhZ: VgauzvAfkJ, d: 08:37:46, a: 767, b: 767, }"] [row2="{ ahRhZ: zt7TwiP9jHAvoSxb1d, d: 08:37:46, a: 767, b: 767, c: KC40rgtkgv, }"] [2025/04/06 08:25:51.780 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ b: 768, c: o, ahRhZ: VgauzvAfkJ, d: 08:37:47, a: 768, }"] [row2="{ b: 768, c: zGuw0sSH, ahRhZ: jgf5bRc3P5bkz9tkGNctY5LXHiOXsg3LTWX, d: 08:37:47, a: 768, }"] [2025/04/06 08:25:51.531 +08:00] [INFO] [file.go:114] ["succeed to download and sort redo logs"] [type=row] [duration=2.05798164s] [2025/04/06 08:25:51.532 +08:00] [ERROR] [redo.go:272] ["ignore unsupported DDL"] [ddl="{\"StartTs\":457152690821267492,\"CommitTs\":457152690834374661,\"Query\":\"ALTER TABLE `consistent_replicate_storage_file`.`t1` EXCHANGE PARTITION `p3` WITH TABLE `consistent_replicate_storage_file`.`t2`\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"Schema\":\"consistent_replicate_storage_file\",\"Table\":\"t1\",\"TableID\":118,\"IsPartition\":true},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":42,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false,\"BDRRole\":\"\",\"SQLMode\":0}"] [2025/04/06 08:25:51.881 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ b: 769, c: o, ahRhZ: VgauzvAfkJ, d: 08:37:48, a: 769, }"] [row2="{ d: 08:37:48, a: 769, b: 769, c: Jif, ahRhZ: wVa8HLQI, }"] [2025/04/06 08:25:51.981 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ b: 770, c: o, ahRhZ: VgauzvAfkJ, d: 08:37:50, a: 770, }"] [row2="{ a: 770, b: 770, c: 2IJb0l, ahRhZ: KuZj3uCN0P157LAfN7ZkOqWOSGV4ZqYf, d: 08:37:50, }"] [2025/04/06 08:25:52.081 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ d: 08:37:51, a: 771, b: 771, c: o, ahRhZ: VgauzvAfkJ, }"] [row2="{ d: 08:37:51, a: 771, b: 771, c: fR, ahRhZ: 483pbnIaqNA1kWFXij7UljHhVJXXJgHM, }"] Verifying downstream PD is started... [2025/04/06 08:25:52.181 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ c: o, ahRhZ: VgauzvAfkJ, d: 08:37:52, a: 772, b: 772, }"] [row2="{ a: 772, b: 772, c: VUonclJ7, ahRhZ: 4GwcFSdbg60DQfk879AeEStpTvqEr9ngFb, d: 08:37:52, }"] [2025/04/06 08:25:52.281 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ ahRhZ: VgauzvAfkJ, d: 08:37:53, a: 773, b: 773, c: o, }"] [row2="{ d: 08:37:53, a: 773, b: 773, c: CuFn9wnSG0, ahRhZ: 0FpwpWubEEGarx51q4yPd77T582x, }"] [2025/04/06 08:25:52.382 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 774, b: 774, c: o, ahRhZ: VgauzvAfkJ, d: 08:37:54, }"] [row2="{ a: 774, b: 774, c: 0LkUZzee, ahRhZ: 9Q1T8POvdAEty6VXDJuwsnIuhfJ2G, d: 08:37:54, }"] [2025/04/06 08:25:52.482 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 775, b: 775, c: o, ahRhZ: VgauzvAfkJ, d: 08:37:55, }"] [row2="{ a: 775, b: 775, c: QV1Rkl, ahRhZ: u1HHyvBZK2Ee0wcEpTMFozAqcHRsPK34GFSn, d: 08:37:55, }"] \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } [2025/04/06 08:25:52.582 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 776, b: 776, c: o, ahRhZ: VgauzvAfkJ, d: 08:37:56, }"] [row2="{ ahRhZ: Z72xsAVSCKdZOLXj8IIgQm, d: 08:37:56, a: 776, b: 776, c: 4fRXf42, }"] [2025/04/06 08:25:52.682 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 777, b: 777, c: o, ahRhZ: VgauzvAfkJ, d: 08:37:57, }"] [row2="{ a: 777, b: 777, c: ifv, ahRhZ: sXkvjdXt, d: 08:37:57, }"] [2025/04/06 08:25:52.782 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 778, b: 778, c: o, ahRhZ: VgauzvAfkJ, d: 08:37:58, }"] [row2="{ a: 778, b: 778, c: kSDBxvjGJc, ahRhZ: 2A6hcYqQ877FjT5kj2EkpLd, d: 08:37:58, }"] [Pipeline] // timeout [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-periodics_cdc_integration_mysql_test-234/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [2025/04/06 08:25:52.883 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ c: o, ahRhZ: VgauzvAfkJ, d: 08:37:59, a: 779, b: 779, }"] [row2="{ a: 779, b: 779, c: FQM3FdPo, ahRhZ: kEjRdnjmsCIMVKL, d: 08:37:59, }"] [2025/04/06 08:25:52.983 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ d: 08:38:00, a: 780, b: 780, c: o, ahRhZ: VgauzvAfkJ, }"] [row2="{ a: 780, b: 780, c: SlODDdhf, ahRhZ: sa, d: 08:38:00, }"] [2025/04/06 08:25:53.083 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 781, b: 781, c: o, ahRhZ: VgauzvAfkJ, d: 08:38:01, }"] [row2="{ a: 781, b: 781, c: uUctYyn57, ahRhZ: 7Cx, d: 08:38:01, }"] Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release [Pipeline] // dir [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [2025/04/06 08:25:53.045 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [span="{tableID:126, startKey:7480000000000000ff7e5f720000000000fa, endKey:7480000000000000ff7e5f730000000000fa}"] [checkpointTs=457152697741869057] [2025/04/06 08:25:53.045 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [span="{tableID:126, startKey:7480000000000000ff7e5f720000000000fa, endKey:7480000000000000ff7e5f730000000000fa}"] [checkpointTs=457152697741869057] [Pipeline] // withEnv [Pipeline] } + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_cdc_cluster.cli.12555.out cli changefeed create --start-ts=457152696683593731 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ --server http://127.0.0.1:8301 --config=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/multi_cdc_cluster/conf/changefeed2.toml nonempty select id, val from test.availability1 where id=4 and val=4 run task successfully test_stop_processor pass [Pipeline] // node [Pipeline] } [2025/04/06 08:25:53.548 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [span="{tableID:128, startKey:7480000000000000ff805f720000000000fa, endKey:7480000000000000ff805f730000000000fa}"] [checkpointTs=457152697741869057] [2025/04/06 08:25:53.548 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [span="{tableID:128, startKey:7480000000000000ff805f720000000000fa, endKey:7480000000000000ff805f730000000000fa}"] [checkpointTs=457152697741869057] [Pipeline] // podTemplate [2025/04/06 08:25:53.183 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ d: 08:38:02, a: 782, b: 782, c: o, ahRhZ: VgauzvAfkJ, }"] [row2="{ a: 782, b: 782, c: Yf7MTpol7H, ahRhZ: QPZXqlFw1X8MQZa5l66EbfBxTJRuLjOgBWMvBlGU, d: 08:38:02, }"] [2025/04/06 08:25:53.284 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 783, b: 783, c: o, ahRhZ: VgauzvAfkJ, d: 08:38:03, }"] [row2="{ d: 08:38:03, a: 783, b: 783, c: MbyOQPClm, ahRhZ: yZ5HooGiGIG7Et, }"] [2025/04/06 08:25:53.384 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 784, b: 784, c: o, ahRhZ: VgauzvAfkJ, d: 08:38:04, }"] [row2="{ d: 08:38:04, a: 784, b: 784, c: rC, ahRhZ: 3, }"] [2025/04/06 08:25:53.484 +08:00] [WARN] [diff.go:937] ["find different row"] [column=ahRhZ] [row1="{ ahRhZ: VgauzvAfkJ, d: 08:38:05, a: 785, b: 785, c: o, }"] [row2="{ ahRhZ: ooWj6qjOyTJElvGw3SpnCPJIwmO8Rtj1St3qqMZ6CW, d: 08:38:05, a: 785, b: 785, c: tyP, }"] [2025/04/06 08:25:53.584 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 786, b: 786, c: o, ahRhZ: VgauzvAfkJ, d: 08:38:06, }"] [row2="{ a: 786, b: 786, c: L391olunIM, ahRhZ: JZqXS4B3ZYvtvegRxoBtILCibDirV, d: 08:38:06, }"] [Pipeline] } Create changefeed successfully! ID: 3ef7810a-a90b-4352-b334-40271ee521d9 Info: {"upstream_id":7489989742764661549,"namespace":"default","id":"3ef7810a-a90b-4352-b334-40271ee521d9","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-06T08:25:53.704365186+08:00","start_ts":457152696683593731,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["test.multi_cdc2"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457152696683593731,"checkpoint_ts":457152696683593731,"checkpoint_time":"2025-04-06 08:25:42.012"} PASS [Pipeline] // withEnv wait process cdc.test exit for 1-th time... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release [Pipeline] } [2025/04/06 08:25:53.649 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [span="{tableID:123, startKey:7480000000000000ff7b5f720000000000fa, endKey:7480000000000000ff7b5f730000000000fa}"] [checkpointTs=457152697741869057] [2025/04/06 08:25:53.649 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [span="{tableID:123, startKey:7480000000000000ff7b5f720000000000fa, endKey:7480000000000000ff7b5f730000000000fa}"] [checkpointTs=457152697741869057] [2025/04/06 08:25:53.749 +08:00] [INFO] [redo.go:223] ["apply redo log finishes"] [appliedLogCount=5008] [appliedDDLCount=0] [currentCheckpoint=457152697741869057] [2025/04/06 08:25:53.749 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=0] [2025/04/06 08:25:53.749 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=6] [2025/04/06 08:25:53.749 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=3] [2025/04/06 08:25:53.749 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=12] [2025/04/06 08:25:53.749 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=5] [2025/04/06 08:25:53.749 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=9] [2025/04/06 08:25:53.749 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=11] [2025/04/06 08:25:53.749 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=4] [2025/04/06 08:25:53.749 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=14] [2025/04/06 08:25:53.749 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=13] [2025/04/06 08:25:53.749 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=8] [2025/04/06 08:25:53.749 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=7] [2025/04/06 08:25:53.749 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=15] [2025/04/06 08:25:53.749 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=2] [2025/04/06 08:25:53.749 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=1] [2025/04/06 08:25:53.749 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=10] Apply redo log successfully [2025/04/06 08:25:53.684 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ ahRhZ: VgauzvAfkJ, d: 08:38:07, a: 787, b: 787, c: o, }"] [row2="{ a: 787, b: 787, c: ImUbI, ahRhZ: cYbQ, d: 08:38:07, }"] [Pipeline] // stage [2025/04/06 08:25:53.785 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ d: 08:38:08, a: 788, b: 788, c: o, ahRhZ: VgauzvAfkJ, }"] [row2="{ a: 788, b: 788, c: 1FrG, ahRhZ: EGLgSp0bkEtUkNAmhtR, d: 08:38:08, }"] [2025/04/06 08:25:53.885 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ ahRhZ: VgauzvAfkJ, d: 08:38:09, a: 789, b: 789, c: o, }"] [row2="{ d: 08:38:09, a: 789, b: 789, c: 7x94EDS, ahRhZ: GbBj6VZB8yo6z6kD5ctYpZDi5R9RP5DXN, }"] [Pipeline] } coverage: 2.4% of statements in github.com/pingcap/tiflow/... wait process cdc.test exit for 2-th time... [2025/04/06 08:25:53.985 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ c: o, ahRhZ: VgauzvAfkJ, d: 08:38:12, a: 790, b: 790, }"] [row2="{ a: 790, b: 790, c: Shp8a, ahRhZ: v74gzHybX, d: 08:38:12, }"] [2025/04/06 08:25:54.085 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ c: o, ahRhZ: VgauzvAfkJ, d: 08:38:13, a: 791, b: 791, }"] [row2="{ b: 791, c: tSGcMO91oA, ahRhZ: uoVxaceRyet7oIf9, d: 08:38:13, a: 791, }"] [2025/04/06 08:25:54.185 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 792, b: 792, c: o, ahRhZ: VgauzvAfkJ, d: 08:38:14, }"] [row2="{ a: 792, b: 792, c: gbLY0T0woC, ahRhZ: IJqujGNiwgWcy88ag, d: 08:38:14, }"] [2025/04/06 08:25:54.286 +08:00] [WARN] [diff.go:937] ["find different row"] [column=ahRhZ] [row1="{ ahRhZ: VgauzvAfkJ, d: 08:38:15, a: 793, b: 793, c: o, }"] [row2="{ c: LveV60, ahRhZ: kpE7Y4cg7TmG4BaJgOs6jv3zj, d: 08:38:15, a: 793, b: 793, }"] [2025/04/06 08:25:54.386 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 794, b: 794, c: o, ahRhZ: VgauzvAfkJ, d: 08:38:16, }"] [row2="{ a: 794, b: 794, c: NhZ, ahRhZ: FnC7EchpQiC25qj5LWYfRzDlZz3QFC, d: 08:38:16, }"] [2025/04/06 08:25:54.486 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ b: 796, c: o, ahRhZ: VgauzvAfkJ, d: 08:38:18, a: 796, }"] [row2="{ c: nhbfpNOcCE, ahRhZ: Cy427tvf301V4Df, d: 08:38:18, a: 796, b: 796, }"] start tidb cluster in /tmp/tidb_cdc_test/savepoint Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/http_api_tls_with_user_auth/run.sh using Sink-Type: mysql... <<================= WARNING: Running pip install with root privileges is generally not a good idea. Try `__main__.py install --user` instead. Requirement already up-to-date: requests==2.26.0 in /usr/local/lib/python3.6/site-packages Requirement already up-to-date: idna<4,>=2.5; python_version >= "3" in /usr/local/lib/python3.6/site-packages (from requests==2.26.0) Requirement already up-to-date: charset-normalizer~=2.0.0; python_version >= "3" in /usr/local/lib/python3.6/site-packages (from requests==2.26.0) Requirement already up-to-date: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.6/site-packages (from requests==2.26.0) Requirement already up-to-date: certifi>=2017.4.17 in /usr/local/lib/python3.6/site-packages (from requests==2.26.0) The 1 times to try to start tidb cluster... [2025/04/06 08:25:54.586 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ c: o, ahRhZ: VgauzvAfkJ, d: 08:38:19, a: 797, b: 797, }"] [row2="{ a: 797, b: 797, c: e6, ahRhZ: EMft5EFwd0ck7BqVBvz5ttlxWm6xIrtxiu4K, d: 08:38:19, }"] [2025/04/06 08:25:54.687 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ b: 798, c: o, ahRhZ: VgauzvAfkJ, d: 08:38:20, a: 798, }"] [row2="{ a: 798, b: 798, c: B, ahRhZ: F4DmENsBkV2uYABH1ODlUuP7wnr5, d: 08:38:20, }"] check diff successfully [Sun Apr 6 08:25:54 CST 2025] <<<<<< run test case consistent_replicate_storage_file success! >>>>>> count(*) 5000 wait process cdc.test exit for 3-th time... process cdc.test already exit [Sun Apr 6 08:25:54 CST 2025] <<<<<< run test case availability success! >>>>>> [2025/04/06 08:25:54.787 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ c: o, ahRhZ: VgauzvAfkJ, d: 08:38:21, a: 799, b: 799, }"] [row2="{ b: 799, c: cgB04, ahRhZ: oSK3Ojlnh9, d: 08:38:21, a: 799, }"] [2025/04/06 08:25:54.887 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 800, b: 800, c: o, ahRhZ: VgauzvAfkJ, d: 08:38:22, }"] [row2="{ a: 800, b: 800, c: 0cy, ahRhZ: BpzGgwu62fbjwFdmx7fvg3rqrhgzSwTVwwim1, d: 08:38:22, }"] [2025/04/06 08:25:54.987 +08:00] [WARN] [diff.go:937] ["find different row"] [column=ahRhZ] [row1="{ d: 08:38:23, a: 801, b: 801, c: o, ahRhZ: VgauzvAfkJ, }"] [row2="{ a: 801, b: 801, c: 7jz, ahRhZ: tlZVwCI4sHbkK0KAPqiL9u, d: 08:38:23, }"] Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2025/04/06 08:25:55.087 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ ahRhZ: VgauzvAfkJ, d: 08:38:24, a: 802, b: 802, c: o, }"] [row2="{ ahRhZ: ZyuYJRPJR8ylsUY8zHjRibOJ3jLVFK01u81kVfJig36, d: 08:38:24, a: 802, b: 802, c: CZ, }"] [2025/04/06 08:25:55.188 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 803, b: 803, c: o, ahRhZ: VgauzvAfkJ, d: 08:38:25, }"] [row2="{ a: 803, b: 803, c: EGejv3hgh, ahRhZ: 8ZdvB3fz0qALqwqcNJWUacuuePrfAzw6ms4BWkK, d: 08:38:25, }"] [2025/04/06 08:25:55.288 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 804, b: 804, c: o, ahRhZ: VgauzvAfkJ, d: 08:38:26, }"] [row2="{ c: VFibPm1m, ahRhZ: 264Rp9V5JD3CFGab94EFpMSogHt0lMXlDVU, d: 08:38:26, a: 804, b: 804, }"] [2025/04/06 08:25:55.388 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 805, b: 805, c: o, ahRhZ: VgauzvAfkJ, d: 08:38:27, }"] [row2="{ a: 805, b: 805, c: 89, ahRhZ: FirCmv8SgOxTctzHf2T3Alp2SJlrBAqt8euPJ3k, d: 08:38:27, }"] [2025/04/06 08:25:55.488 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ c: o, ahRhZ: VgauzvAfkJ, d: 08:38:28, a: 806, b: 806, }"] [row2="{ a: 806, b: 806, c: PY0niUz3, ahRhZ: U74N9A4XQAPsh, d: 08:38:28, }"] Starting Upstream TiDB... [2025/04/06 08:25:55.588 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 807, b: 807, c: o, ahRhZ: VgauzvAfkJ, d: 08:38:29, }"] [row2="{ d: 08:38:29, a: 807, b: 807, c: r347w, ahRhZ: 0Lmui2EsXeA7YbeBQ1Lao, }"] [2025/04/06 08:25:55.689 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ c: o, ahRhZ: VgauzvAfkJ, d: 08:38:30, a: 808, b: 808, }"] [row2="{ d: 08:38:30, a: 808, b: 808, c: AFnCrJ, ahRhZ: iCJzfT1MlzzASzM6nkqkxkMSpD5ihK, }"] Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2025/04/06 08:25:55.789 +08:00] [WARN] [diff.go:940] ["target had superfluous data"] [row="{ ahRhZ: OYA, d: 08:38:31, a: 809, b: 809, c: YQ2uRK3Ryr, }"] [2025/04/06 08:25:55.889 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ c: o, ahRhZ: VgauzvAfkJ, d: 08:38:32, a: 810, b: 810, }"] [row2="{ a: 810, b: 810, c: sWk, ahRhZ: SulXACkhfiUxkYZmpQz7HEW711tx7l2HBnoJu1b9Z, d: 08:38:32, }"] [2025/04/06 08:25:55.989 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ b: 811, c: o, ahRhZ: VgauzvAfkJ, d: 08:38:33, a: 811, }"] [row2="{ ahRhZ: SCVvcnpdCJxqTWIQUvc6IbhHSvT9ppctfjDXHx1ZCGVAh, d: 08:38:33, a: 811, b: 811, c: G68vf1, }"] [2025/04/06 08:25:56.089 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ c: o, ahRhZ: VgauzvAfkJ, d: 08:38:34, a: 812, b: 812, }"] [row2="{ b: 812, c: KxmjlAHeR, ahRhZ: C9mHgJeRKL8gzGhZLIc6bfL5REn5XdO57ZTszo1, d: 08:38:34, a: 812, }"] [2025/04/06 08:25:56.189 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 813, b: 813, c: o, ahRhZ: VgauzvAfkJ, d: 08:38:35, }"] [row2="{ ahRhZ: 07RgjwEeNzbBuAmUwM6D3wXHgHPbI4wi, d: 08:38:35, a: 813, b: 813, c: UDdQTkKcgG, }"] [2025/04/06 08:25:56.290 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 814, b: 814, c: o, ahRhZ: VgauzvAfkJ, d: 08:38:36, }"] [row2="{ b: 814, c: Wze1OdEYEc, ahRhZ: nofXkWtVYkHc8PtduArIGFCzcPcFTkV2nAuo0FF8sa, d: 08:38:36, a: 814, }"] [2025/04/06 08:25:56.390 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 815, b: 815, c: o, ahRhZ: VgauzvAfkJ, d: 08:38:37, }"] [row2="{ d: 08:38:37, a: 815, b: 815, c: kkdO, ahRhZ: sf3, }"] [2025/04/06 08:25:56.490 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 816, b: 816, c: o, ahRhZ: VgauzvAfkJ, d: 08:38:38, }"] [row2="{ b: 816, c: RLJh, ahRhZ: uON8VfoWyPxvRBzdTcyUyXNRU, d: 08:38:38, a: 816, }"] [2025/04/06 08:25:56.590 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ b: 817, c: o, ahRhZ: VgauzvAfkJ, d: 08:38:39, a: 817, }"] [row2="{ a: 817, b: 817, c: 8102EbWhJ, ahRhZ: sybpBjKGMWE5, d: 08:38:39, }"] [2025/04/06 08:25:56.690 +08:00] [WARN] [diff.go:937] ["find different row"] [column=ahRhZ] [row1="{ a: 818, b: 818, c: o, ahRhZ: VgauzvAfkJ, d: 08:38:40, }"] [row2="{ c: ijPAAi0p, ahRhZ: tj9cMehdUYr8csGGDLLJIQ6VLhZ4eW, d: 08:38:40, a: 818, b: 818, }"] [2025/04/06 08:25:56.791 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ c: o, ahRhZ: VgauzvAfkJ, d: 08:38:41, a: 819, b: 819, }"] [row2="{ ahRhZ: RxUxbjHukSreWi6AeYTlunjct, d: 08:38:41, a: 819, b: 819, c: b, }"] [2025/04/06 08:25:56.891 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 820, b: 820, c: o, ahRhZ: VgauzvAfkJ, d: 08:38:42, }"] [row2="{ a: 820, b: 820, c: J13, ahRhZ: wnOMSugBhUsFGP, d: 08:38:42, }"] [2025/04/06 08:25:56.991 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ d: 08:38:43, a: 821, b: 821, c: o, ahRhZ: VgauzvAfkJ, }"] [row2="{ a: 821, b: 821, c: ZPyYBh, ahRhZ: 5q9l8KzA, d: 08:38:43, }"] + set +x [2025/04/06 08:25:57.091 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ d: 08:38:44, a: 822, b: 822, c: o, ahRhZ: VgauzvAfkJ, }"] [row2="{ ahRhZ: 5P3, d: 08:38:44, a: 822, b: 822, c: KwQF, }"] [2025/04/06 08:25:57.192 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ d: 08:38:45, a: 823, b: 823, c: o, ahRhZ: VgauzvAfkJ, }"] [row2="{ d: 08:38:45, a: 823, b: 823, c: Zy6, ahRhZ: OKGgFLgvnK8ERnupjdf3Uio5riDTM5yCuO9ufLUaIIM01, }"] table test.multi_cdc1 exists table test.multi_cdc2 exists ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2025/04/06 08:25:57.292 +08:00] [WARN] [diff.go:937] ["find different row"] [column=ahRhZ] [row1="{ a: 824, b: 824, c: o, ahRhZ: VgauzvAfkJ, d: 08:38:46, }"] [row2="{ a: 824, b: 824, c: gCG, ahRhZ: fQi1p, d: 08:38:46, }"] [2025/04/06 08:25:57.392 +08:00] [WARN] [diff.go:940] ["target had superfluous data"] [row="{ a: 825, b: 825, c: BT67Vav8vO, ahRhZ: lawNMdvVlib8XP, d: 08:38:47, }"] [2025/04/06 08:25:57.492 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ c: o, ahRhZ: VgauzvAfkJ, d: 08:38:48, a: 826, b: 826, }"] [row2="{ ahRhZ: BgXkiqxZyWn4LAtXXhHuK0y4nIqy, d: 08:38:48, a: 826, b: 826, c: dLu4qG, }"] check diff failed 1-th time, retry later [2025/04/06 08:25:57.592 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 827, b: 827, c: o, ahRhZ: VgauzvAfkJ, d: 08:38:49, }"] [row2="{ a: 827, b: 827, c: y4DqUb, ahRhZ: y8qJ9cvhGu, d: 08:38:49, }"] [2025/04/06 08:25:57.693 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 828, b: 828, c: o, ahRhZ: VgauzvAfkJ, d: 08:38:50, }"] [row2="{ a: 828, b: 828, c: 2hmt8, ahRhZ: DxP13qRKOHoRHFKF4FH8j2MZPDv2t5uJ6k4ijABdeXQ, d: 08:38:50, }"] [2025/04/06 08:25:57.793 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 829, b: 829, c: o, ahRhZ: VgauzvAfkJ, d: 08:38:51, }"] [row2="{ ahRhZ: kHOYqX6EatdNgBtgofXi4, d: 08:38:51, a: 829, b: 829, c: hKyrKl, }"] [2025/04/06 08:25:57.893 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 830, b: 830, c: o, ahRhZ: VgauzvAfkJ, d: 08:38:53, }"] [row2="{ ahRhZ: dS3AtSUYFbUrbYCT0Rf, d: 08:38:53, a: 830, b: 830, c: b, }"] [2025/04/06 08:25:57.993 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 831, b: 831, c: o, ahRhZ: VgauzvAfkJ, d: 08:38:54, }"] [row2="{ d: 08:38:54, a: 831, b: 831, c: favWMr7JA, ahRhZ: OQWcJgYf73rWowkseLseP7uah, }"] [2025/04/06 08:25:58.093 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 832, b: 832, c: o, ahRhZ: VgauzvAfkJ, d: 08:38:55, }"] [row2="{ ahRhZ: R871YsMF7fl2XNdKW5uxu7, d: 08:38:55, a: 832, b: 832, c: 76wg, }"] [2025/04/06 08:25:58.193 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ d: 08:38:56, a: 833, b: 833, c: o, ahRhZ: VgauzvAfkJ, }"] [row2="{ d: 08:38:56, a: 833, b: 833, c: PyivrhtXY, ahRhZ: TjjQ198gfKpI9dyiPEQLv0dNA3lpmSsg9CDLWyB, }"] [2025/04/06 08:25:58.294 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 834, b: 834, c: o, ahRhZ: VgauzvAfkJ, d: 08:38:57, }"] [row2="{ d: 08:38:57, a: 834, b: 834, c: g0ev9y, ahRhZ: RQVQ8lcJrN2z92OLFNSUxVV, }"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2025/04/06 08:25:58.394 +08:00] [WARN] [diff.go:937] ["find different row"] [column=ahRhZ] [row1="{ b: 835, c: o, ahRhZ: VgauzvAfkJ, d: 08:41:37, a: 835, }"] [row2="{ a: 835, b: 835, c: WC6AvWhIP, ahRhZ: Ekdm9nWoPnbVaYn0pDIjYxu3T2mWQl6gP, d: 08:38:58, }"] [2025/04/06 08:25:58.494 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ c: o, ahRhZ: VgauzvAfkJ, d: 08:38:59, a: 836, b: 836, }"] [row2="{ a: 836, b: 836, c: YdTvtA3t, ahRhZ: b1DPzBNvLirvVpHsz29, d: 08:38:59, }"] Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release [2025/04/06 08:25:58.594 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ ahRhZ: VgauzvAfkJ, d: 08:39:00, a: 837, b: 837, c: o, }"] [row2="{ b: 837, c: GfWFvWXjQ, ahRhZ: LPtZmE5aIaZr0Sv71uwJlSWKJepunguuX8GPyEMFt2Lc, d: 08:39:00, a: 837, }"] [2025/04/06 08:25:58.694 +08:00] [WARN] [diff.go:937] ["find different row"] [column=ahRhZ] [row1="{ a: 838, b: 838, c: o, ahRhZ: VgauzvAfkJ, d: 08:39:01, }"] [row2="{ b: 838, c: VbO, ahRhZ: 832gcskoQXbquFRM1qp, d: 08:39:01, a: 838, }"] [2025/04/06 08:25:58.795 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 839, b: 839, c: o, ahRhZ: VgauzvAfkJ, d: 08:39:02, }"] [row2="{ b: 839, c: IOOxDupS5t, ahRhZ: ZD6GdCkOV5, d: 08:39:02, a: 839, }"] start tidb cluster in /tmp/tidb_cdc_test/http_api_tls_with_user_auth Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... [2025/04/06 08:25:58.895 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 840, b: 840, c: o, ahRhZ: VgauzvAfkJ, d: 08:39:04, }"] [row2="{ d: 08:39:04, a: 840, b: 840, c: 3gJsYT8, ahRhZ: Qw8MTNZYp8qmlcrX8l0dQTg4wBlh60cW, }"] [2025/04/06 08:25:58.995 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ ahRhZ: VgauzvAfkJ, d: 08:39:05, a: 841, b: 841, c: o, }"] [row2="{ a: 841, b: 841, c: VyA, ahRhZ: gSXoI7WYvhwlfteDcBAS6gRDGBWDcBAa9xpGHIyl, d: 08:39:05, }"] [2025/04/06 08:25:59.095 +08:00] [WARN] [diff.go:937] ["find different row"] [column=ahRhZ] [row1="{ a: 842, b: 842, c: o, ahRhZ: VgauzvAfkJ, d: 08:39:06, }"] [row2="{ c: Z, ahRhZ: mLcOpeepXcuS8bcnts, d: 08:39:06, a: 842, b: 842, }"] [2025/04/06 08:25:59.195 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 843, b: 843, c: o, ahRhZ: VgauzvAfkJ, d: 08:39:07, }"] [row2="{ a: 843, b: 843, c: 4Sse, ahRhZ: 84CneBLBa7s2IuT9oM3xRtVv, d: 08:39:07, }"] [2025/04/06 08:25:59.296 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ b: 844, c: o, ahRhZ: VgauzvAfkJ, d: 08:39:08, a: 844, }"] [row2="{ a: 844, b: 844, c: a4, ahRhZ: BUC6X8eZKje7BG4Vy4kXQoyx, d: 08:39:08, }"] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489989820831948132 TiDB Cluster ID. tikv_gc_leader_uuid 65821f596a80005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-x-pbmg2, pid:10664, start at 2025-04-06 08:25:57.943050985 +0800 CST m=+3.070036673 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:27:57.954 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:25:57.944 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:15:57.944 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489989820831948132 TiDB Cluster ID. tikv_gc_leader_uuid 65821f596a80005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-x-pbmg2, pid:10664, start at 2025-04-06 08:25:57.943050985 +0800 CST m=+3.070036673 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:27:57.954 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:25:57.944 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:15:57.944 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489989821504113322 TiDB Cluster ID. tikv_gc_leader_uuid 65821f597e00002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-x-pbmg2, pid:10721, start at 2025-04-06 08:25:58.036839517 +0800 CST m=+3.096361433 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:27:58.049 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:25:58.008 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:15:58.008 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/owner_remove_table_error/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/owner_remove_table_error/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/owner_remove_table_error/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/owner_remove_table_error/tiflash/db/proxy"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/owner_remove_table_error/tiflash/log/proxy.log"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [2025/04/06 08:25:59.396 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 845, b: 845, c: o, ahRhZ: VgauzvAfkJ, d: 08:39:09, }"] [row2="{ ahRhZ: nFhrn753wOMKMbFLgvDx, d: 08:39:09, a: 845, b: 845, c: zCmpuSVvIR, }"] [2025/04/06 08:25:59.496 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ ahRhZ: VgauzvAfkJ, d: 08:39:10, a: 846, b: 846, c: o, }"] [row2="{ b: 846, c: Fvejr5Ewtd, ahRhZ: RWmOHX0HUuQtmB7ljgC9gnGVgp9KNLhc, d: 08:39:10, a: 846, }"] [2025/04/06 08:25:59.596 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ d: 08:39:11, a: 847, b: 847, c: o, ahRhZ: VgauzvAfkJ, }"] [row2="{ a: 847, b: 847, c: 55L9VqTgjp, ahRhZ: fA, d: 08:39:11, }"] [2025/04/06 08:25:59.696 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ d: 08:39:12, a: 848, b: 848, c: o, ahRhZ: VgauzvAfkJ, }"] [row2="{ d: 08:39:12, a: 848, b: 848, c: f1, ahRhZ: asWGuHvYMslTutejnAdVhc2CGH4l2zP5XRnwHVp, }"] [2025/04/06 08:25:59.796 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ c: o, ahRhZ: VgauzvAfkJ, d: 08:39:13, a: 849, b: 849, }"] [row2="{ a: 849, b: 849, c: BUXMy, ahRhZ: vhTpNzO1YKE5UrnZW5ld6cF2BM, d: 08:39:13, }"] Starting Upstream TiDB... [2025/04/06 08:25:59.897 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ ahRhZ: VgauzvAfkJ, d: 08:39:14, a: 850, b: 850, c: o, }"] [row2="{ a: 850, b: 850, c: yIW7Y, ahRhZ: Lm0NXoM, d: 08:39:14, }"] [2025/04/06 08:25:59.997 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 851, b: 851, c: o, ahRhZ: VgauzvAfkJ, d: 08:39:15, }"] [row2="{ b: 851, c: i0, ahRhZ: iCns1UFpJc03fgKMr0LCs4aCCzYwRdBEBfZLRU, d: 08:39:15, a: 851, }"] check diff successfully Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled False If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489989827991870654 TiDB Cluster ID. tikv_gc_leader_uuid 65821f598440016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-x-hbn5c, pid:32183, start at 2025-04-06 08:25:58.070023536 +0800 CST m=+2.331567433 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:27:58.077 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:25:58.083 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:15:58.083 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled False If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489989827991870654 TiDB Cluster ID. tikv_gc_leader_uuid 65821f598440016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-x-hbn5c, pid:32183, start at 2025-04-06 08:25:58.070023536 +0800 CST m=+2.331567433 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:27:58.077 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:25:58.083 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:15:58.083 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled False If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489989833109698692 TiDB Cluster ID. tikv_gc_leader_uuid 65821f599cc000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-x-hbn5c, pid:32255, start at 2025-04-06 08:25:58.179787776 +0800 CST m=+2.346908254 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:27:58.188 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:25:58.181 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:15:58.181 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/ci_collation_compatibility/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/ci_collation_compatibility/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/ci_collation_compatibility/tiflash/log/proxy.log"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/ci_collation_compatibility/tiflash-proxy.toml"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/ci_collation_compatibility/tiflash/db/proxy"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [2025/04/06 08:26:00.097 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 852, b: 852, c: o, ahRhZ: VgauzvAfkJ, d: 08:39:16, }"] [row2="{ a: 852, b: 852, c: 5u, ahRhZ: zirMdaBpl0yGfFv38QQfjrnR4vljeAtqE, d: 08:39:16, }"] [2025/04/06 08:26:00.197 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ c: o, ahRhZ: VgauzvAfkJ, d: 08:39:17, a: 853, b: 853, }"] [row2="{ ahRhZ: aZ3Q28uLa5o24SiE8tGomXCRHU8gE8ZC, d: 08:39:17, a: 853, b: 853, c: 8gJLfhMnP7, }"] [2025/04/06 08:26:00.297 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 854, b: 854, c: o, ahRhZ: VgauzvAfkJ, d: 08:39:18, }"] [row2="{ a: 854, b: 854, c: t, ahRhZ: MMmgv8DkKKWEtohqa3cU9MLGJoUmlSVig9EZn65, d: 08:39:18, }"] wait process cdc.test exit for 1-th time... [2025/04/06 08:26:00.398 +08:00] [WARN] [diff.go:937] ["find different row"] [column=ahRhZ] [row1="{ b: 855, c: o, ahRhZ: VgauzvAfkJ, d: 08:39:19, a: 855, }"] [row2="{ c: ercHdYRV, ahRhZ: zJP0V5OQkfjVD2NyV, d: 08:39:19, a: 855, b: 855, }"] [2025/04/06 08:26:00.498 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 856, b: 856, c: o, ahRhZ: VgauzvAfkJ, d: 08:39:20, }"] [row2="{ a: 856, b: 856, c: 2vthHCo, ahRhZ: nX1lNZKj, d: 08:39:20, }"] [2025/04/06 08:26:00.598 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 857, b: 857, c: o, ahRhZ: VgauzvAfkJ, d: 08:39:21, }"] [row2="{ ahRhZ: t5xt7ycwbZqIEnvXIgDGM9nwkuKSKcAs81D5uXnVr1sC0, d: 08:39:21, a: 857, b: 857, c: eccj6, }"] [2025/04/06 08:26:00.698 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ ahRhZ: VgauzvAfkJ, d: 08:39:22, a: 858, b: 858, c: o, }"] [row2="{ a: 858, b: 858, c: px7wT, ahRhZ: PKgBnrHZDq1ZedJlx5ccR7LNH0l2Hsh4PiwgJ, d: 08:39:22, }"] [2025/04/06 08:26:00.798 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ b: 859, c: o, ahRhZ: VgauzvAfkJ, d: 08:39:23, a: 859, }"] [row2="{ c: 6h3Rn, ahRhZ: uL, d: 08:39:23, a: 859, b: 859, }"] [2025/04/06 08:26:00.899 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ ahRhZ: VgauzvAfkJ, d: 08:39:26, a: 860, b: 860, c: o, }"] [row2="{ c: WM5rabk9jC, ahRhZ: usxCIyJlDl4Qq9wuqZz6WsYBoGcw, d: 08:39:26, a: 860, b: 860, }"] [2025/04/06 08:26:00.999 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 861, b: 861, c: o, ahRhZ: VgauzvAfkJ, d: 08:39:27, }"] [row2="{ a: 861, b: 861, c: CI, ahRhZ: 4eEHZG06aoNIDxcRpgyusyMdmQ, d: 08:39:27, }"] wait process cdc.test exit for 2-th time... [2025/04/06 08:26:01.099 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 862, b: 862, c: o, ahRhZ: VgauzvAfkJ, d: 08:39:28, }"] [row2="{ a: 862, b: 862, c: NhBfW, ahRhZ: ldxCPW7j37u0WuomquxNCIQ6mUcsfywmDtypi0dMB, d: 08:39:28, }"] [2025/04/06 08:26:01.199 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ d: 08:39:29, a: 863, b: 863, c: o, ahRhZ: VgauzvAfkJ, }"] [row2="{ a: 863, b: 863, c: tZnwF, ahRhZ: lN1VqGBLsMjTo3aaAXu4zD1zB, d: 08:39:29, }"] [2025/04/06 08:26:01.299 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 864, b: 864, c: o, ahRhZ: VgauzvAfkJ, d: 08:39:30, }"] [row2="{ d: 08:39:30, a: 864, b: 864, c: GT, ahRhZ: tGzsrhzup9DmrMsarR8, }"] wait process cdc.test exit for 3-th time... [2025/04/06 08:26:01.400 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 865, b: 865, c: o, ahRhZ: VgauzvAfkJ, d: 08:39:31, }"] [row2="{ c: iGNgm5C, ahRhZ: dS5AUNOjMqasZSwz1oj9W4fV, d: 08:39:31, a: 865, b: 865, }"] [2025/04/06 08:26:01.500 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ d: 08:39:32, a: 866, b: 866, c: o, ahRhZ: VgauzvAfkJ, }"] [row2="{ a: 866, b: 866, c: 3zB, ahRhZ: FcCQWXRS6l6DaDQuDdUJmzKcQXuaEAHTzrkOqVMMPVqKb, d: 08:39:32, }"] [2025/04/06 08:26:01.600 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 867, b: 867, c: o, ahRhZ: VgauzvAfkJ, d: 08:39:33, }"] [row2="{ d: 08:39:33, a: 867, b: 867, c: dJcRGtY, ahRhZ: 8roX0X5CrO36qWu, }"] [2025/04/06 08:26:01.700 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ b: 868, c: o, ahRhZ: VgauzvAfkJ, d: 08:39:34, a: 868, }"] [row2="{ a: 868, b: 868, c: 3, ahRhZ: w8mPYnBCQ1ZrC, d: 08:39:34, }"] [2025/04/06 08:26:01.800 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 869, b: 869, c: o, ahRhZ: VgauzvAfkJ, d: 08:39:35, }"] [row2="{ b: 869, c: zZMoJ, ahRhZ: sCW, d: 08:39:35, a: 869, }"] cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Sun Apr 6 08:26:01 CST 2025] <<<<<< run test case multi_cdc_cluster success! >>>>>> [Sun Apr 6 08:26:01 CST 2025] <<<<<< START cdc server in owner_remove_table_error case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/OwnerRemoveTableError=1*return(true)' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.owner_remove_table_error.1203912041.out server --log-file /tmp/tidb_cdc_test/owner_remove_table_error/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/owner_remove_table_error/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2025/04/06 08:26:01.900 +08:00] [WARN] [diff.go:937] ["find different row"] [column=ahRhZ] [row1="{ a: 870, b: 870, c: o, ahRhZ: VgauzvAfkJ, d: 08:39:36, }"] [row2="{ a: 870, b: 870, c: eCJ1bY, ahRhZ: clDoyEMTWajnI3hFHOwDbujPH9w5s4vvy9Eci, d: 08:39:36, }"] [2025/04/06 08:26:02.001 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ b: 871, c: o, ahRhZ: VgauzvAfkJ, d: 08:39:37, a: 871, }"] [row2="{ a: 871, b: 871, c: p9pDP2icvA, ahRhZ: 3FAcX1om4XyTm0NCAJOADA4, d: 08:39:37, }"] [2025/04/06 08:26:02.101 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ ahRhZ: VgauzvAfkJ, d: 08:39:38, a: 872, b: 872, c: o, }"] [row2="{ d: 08:39:38, a: 872, b: 872, c: hLEVz, ahRhZ: 5ZwQMCEB7xa6D3MKYSb5bRFz7NKWSjj2, }"] Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ci_collation_compatibility.cli.33523.out cli tso query --pd=http://127.0.0.1:2379 [2025/04/06 08:26:02.201 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ c: o, ahRhZ: VgauzvAfkJ, d: 08:39:39, a: 873, b: 873, }"] [row2="{ a: 873, b: 873, c: X, ahRhZ: YjHa, d: 08:39:39, }"] [2025/04/06 08:26:02.301 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ d: 08:39:40, a: 874, b: 874, c: o, ahRhZ: VgauzvAfkJ, }"] [row2="{ a: 874, b: 874, c: WzAm2wc, ahRhZ: gZuD5h1mZVw6dJTN1bIUzS7, d: 08:39:40, }"] [2025/04/06 08:26:02.401 +08:00] [WARN] [diff.go:937] ["find different row"] [column=ahRhZ] [row1="{ b: 875, c: o, ahRhZ: VgauzvAfkJ, d: 08:39:41, a: 875, }"] [row2="{ a: 875, b: 875, c: rL, ahRhZ: iFdPKG1eEBuhlJwjwWhi37EVBh66pD, d: 08:39:41, }"] [2025/04/06 08:26:02.501 +08:00] [WARN] [diff.go:937] ["find different row"] [column=ahRhZ] [row1="{ a: 876, b: 876, c: o, ahRhZ: VgauzvAfkJ, d: 08:39:42, }"] [row2="{ a: 876, b: 876, c: REss1y, ahRhZ: 1vF9PeBnjYDnpzEsCW, d: 08:39:42, }"] [2025/04/06 08:26:02.602 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ c: o, ahRhZ: VgauzvAfkJ, d: 08:39:43, a: 877, b: 877, }"] [row2="{ a: 877, b: 877, c: BlQM, ahRhZ: oEh70GhJByY2rv0ZLX0a7W8jvrs7OpOodvO2a, d: 08:39:43, }"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2025/04/06 08:26:02.702 +08:00] [WARN] [diff.go:937] ["find different row"] [column=ahRhZ] [row1="{ a: 878, b: 878, c: o, ahRhZ: VgauzvAfkJ, d: 08:39:44, }"] [row2="{ a: 878, b: 878, c: MhdLp8Mp, ahRhZ: 5FwLwJEmMlt1dwU, d: 08:39:44, }"] [2025/04/06 08:26:02.802 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ b: 879, c: o, ahRhZ: VgauzvAfkJ, d: 08:39:45, a: 879, }"] [row2="{ a: 879, b: 879, c: ufihg, ahRhZ: KuHZRSyfPec0zCd, d: 08:39:45, }"] [2025/04/06 08:26:02.902 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ ahRhZ: VgauzvAfkJ, d: 08:39:46, a: 880, b: 880, c: o, }"] [row2="{ c: u95QiCv, ahRhZ: 7Fn8eTwb4nWdKC8cCK8tgpr1DpN, d: 08:39:46, a: 880, b: 880, }"] [2025/04/06 08:26:03.002 +08:00] [WARN] [diff.go:937] ["find different row"] [column=ahRhZ] [row1="{ ahRhZ: VgauzvAfkJ, d: 08:39:47, a: 881, b: 881, c: o, }"] [row2="{ c: ThWjDtNNl, ahRhZ: uW441nzvSsHgkBtII70WkwngRQR9zN5tF, d: 08:39:47, a: 881, b: 881, }"] [2025/04/06 08:26:03.102 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ ahRhZ: VgauzvAfkJ, d: 08:39:48, a: 882, b: 882, c: o, }"] [row2="{ d: 08:39:48, a: 882, b: 882, c: qjC, ahRhZ: OLgZNhHcfbbtBU2el1GrlXV30dlTc9eItHzu2RcquH, }"] [2025/04/06 08:26:03.203 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 883, b: 883, c: o, ahRhZ: VgauzvAfkJ, d: 08:39:49, }"] [row2="{ c: oSLBqE, ahRhZ: 7foWBkWTdQlmE1a, d: 08:39:49, a: 883, b: 883, }"] [2025/04/06 08:26:03.303 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ c: o, ahRhZ: VgauzvAfkJ, d: 08:39:50, a: 884, b: 884, }"] [row2="{ c: yQdJQ, ahRhZ: gsSTXPZzF3WSpDu, d: 08:39:50, a: 884, b: 884, }"] [2025/04/06 08:26:03.403 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 885, b: 885, c: o, ahRhZ: VgauzvAfkJ, d: 08:39:51, }"] [row2="{ ahRhZ: 4WJ7, d: 08:39:51, a: 885, b: 885, c: Jyi, }"] [2025/04/06 08:26:03.504 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 886, b: 886, c: o, ahRhZ: VgauzvAfkJ, d: 08:39:52, }"] [row2="{ a: 886, b: 886, c: su7TC, ahRhZ: lC86mo7XNJyFem4uVOuWscYI7PX701Wsw87NKt, d: 08:39:52, }"] [2025/04/06 08:26:03.604 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 887, b: 887, c: o, ahRhZ: VgauzvAfkJ, d: 08:39:53, }"] [row2="{ b: 887, c: U18PRFv, ahRhZ: YairxfrijBkKorop3NRDJUcIPGI4uKwYUreD, d: 08:39:53, a: 887, }"] [2025/04/06 08:26:03.704 +08:00] [WARN] [diff.go:937] ["find different row"] [column=ahRhZ] [row1="{ b: 888, c: o, ahRhZ: VgauzvAfkJ, d: 08:39:54, a: 888, }"] [row2="{ c: LnYYAsy, ahRhZ: OcnogTvWe0gmgrEsn1dzn5yqSYlS, d: 08:39:54, a: 888, b: 888, }"] [2025/04/06 08:26:03.804 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ c: o, ahRhZ: VgauzvAfkJ, d: 08:39:55, a: 889, b: 889, }"] [row2="{ a: 889, b: 889, c: EIgk4hOv, ahRhZ: XvyCIKNiBWKbfyu6z8N3wVt, d: 08:39:55, }"] Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x + tso='457152702049943553 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 457152702049943553 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Sun Apr 6 08:26:03 CST 2025] <<<<<< START cdc server in ci_collation_compatibility case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ci_collation_compatibility.3355633558.out server --log-file /tmp/tidb_cdc_test/ci_collation_compatibility/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ci_collation_compatibility/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2025/04/06 08:26:03.907 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ c: o, ahRhZ: VgauzvAfkJ, d: 08:39:56, a: 890, b: 890, }"] [row2="{ b: 890, c: gU, ahRhZ: ZBUQTEblsdtBIcsopZ8xWxL4BKn1Rqh2WJCNEc0l, d: 08:39:56, a: 890, }"] [2025/04/06 08:26:04.007 +08:00] [WARN] [diff.go:937] ["find different row"] [column=ahRhZ] [row1="{ a: 891, b: 891, c: o, ahRhZ: VgauzvAfkJ, d: 08:39:57, }"] [row2="{ a: 891, b: 891, c: L2mXI61, ahRhZ: HKlxILIcUmNMnkesx, d: 08:39:57, }"] [2025/04/06 08:26:04.107 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 892, b: 892, c: o, ahRhZ: VgauzvAfkJ, d: 08:39:58, }"] [row2="{ ahRhZ: 0t9whHdRpXlR1NCtlvuUEBdM0oqqFB1Mnt, d: 08:39:58, a: 892, b: 892, c: qimITfBG, }"] [2025/04/06 08:26:04.207 +08:00] [WARN] [diff.go:937] ["find different row"] [column=ahRhZ] [row1="{ b: 893, c: o, ahRhZ: VgauzvAfkJ, d: 08:39:59, a: 893, }"] [row2="{ b: 893, c: M8XY1eB, ahRhZ: vyXJiYVvSTo, d: 08:39:59, a: 893, }"] [2025/04/06 08:26:04.307 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 894, b: 894, c: o, ahRhZ: VgauzvAfkJ, d: 08:40:00, }"] [row2="{ ahRhZ: vhFxbBvKP4QwVqe7J9gAtI, d: 08:40:00, a: 894, b: 894, c: 6gL, }"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 06 Apr 2025 00:26:04 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ac2b576d-1463-4478-8800-2d0580d026d1 {"id":"ac2b576d-1463-4478-8800-2d0580d026d1","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899161} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087d476cda ac2b576d-1463-4478-8800-2d0580d026d1 /tidb/cdc/default/default/upstream/7489989820831948132 {"id":7489989820831948132,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' [2025/04/06 08:26:04.408 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 895, b: 895, c: o, ahRhZ: VgauzvAfkJ, d: 08:40:01, }"] [row2="{ c: wNMTV, ahRhZ: 4wZXUjYMEJRtUSHUL, d: 08:40:01, a: 895, b: 895, }"] [2025/04/06 08:26:04.508 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 896, b: 896, c: o, ahRhZ: VgauzvAfkJ, d: 08:40:02, }"] [row2="{ d: 08:40:02, a: 896, b: 896, c: RH8S4C4, ahRhZ: Ojf3CKLbSoNBy9EZRZ9J2d5TwVZchnqSmu0Tw, }"] [2025/04/06 08:26:04.608 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ c: o, ahRhZ: VgauzvAfkJ, d: 08:40:03, a: 897, b: 897, }"] [row2="{ d: 08:40:03, a: 897, b: 897, c: K, ahRhZ: qmpfuPs1lm5Md3JaFp90, }"] + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ac2b576d-1463-4478-8800-2d0580d026d1 {"id":"ac2b576d-1463-4478-8800-2d0580d026d1","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899161} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087d476cda ac2b576d-1463-4478-8800-2d0580d026d1 /tidb/cdc/default/default/upstream/7489989820831948132 {"id":7489989820831948132,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ac2b576d-1463-4478-8800-2d0580d026d1 {"id":"ac2b576d-1463-4478-8800-2d0580d026d1","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899161} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087d476cda ac2b576d-1463-4478-8800-2d0580d026d1 /tidb/cdc/default/default/upstream/7489989820831948132 {"id":7489989820831948132,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489989843854196779 TiDB Cluster ID. tikv_gc_leader_uuid 65821f5debc000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-c-g3td9, pid:13562, start at 2025-04-06 08:26:02.573610229 +0800 CST m=+2.453697679 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:28:02.582 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:26:02.543 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:16:02.543 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489989843854196779 TiDB Cluster ID. tikv_gc_leader_uuid 65821f5debc000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-c-g3td9, pid:13562, start at 2025-04-06 08:26:02.573610229 +0800 CST m=+2.453697679 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:28:02.582 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:26:02.543 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:16:02.543 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489989842091078829 TiDB Cluster ID. tikv_gc_leader_uuid 65821f5e0680007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-c-g3td9, pid:13634, start at 2025-04-06 08:26:02.665400398 +0800 CST m=+2.478026759 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:28:02.676 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:26:02.650 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:16:02.650 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/savepoint/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/savepoint/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/savepoint/tiflash/log/proxy.log"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/savepoint/tiflash-proxy.toml"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/savepoint/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/consistent_replicate_storage_file_large_value/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [2025/04/06 08:26:04.708 +08:00] [WARN] [diff.go:937] ["find different row"] [column=ahRhZ] [row1="{ b: 898, c: o, ahRhZ: VgauzvAfkJ, d: 08:40:04, a: 898, }"] [row2="{ d: 08:40:04, a: 898, b: 898, c: lSx3, ahRhZ: kjbxwXWJ2NZy4dpduCOf, }"] [2025/04/06 08:26:04.808 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 899, b: 899, c: o, ahRhZ: VgauzvAfkJ, d: 08:40:05, }"] [row2="{ a: 899, b: 899, c: zM, ahRhZ: Dh0CYSxe5XmuVxbpBF2gpj79Eh, d: 08:40:05, }"] [2025/04/06 08:26:04.909 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ b: 900, c: o, ahRhZ: VgauzvAfkJ, d: 08:40:06, a: 900, }"] [row2="{ b: 900, c: CyUTQPO, ahRhZ: ZKzHmw2y30JMd67NFcuiJxNmtlVx0cx, d: 08:40:06, a: 900, }"] [2025/04/06 08:26:05.009 +08:00] [WARN] [diff.go:937] ["find different row"] [column=ahRhZ] [row1="{ a: 901, b: 901, c: o, ahRhZ: VgauzvAfkJ, d: 08:40:07, }"] [row2="{ a: 901, b: 901, c: 6tN9FHaiD, ahRhZ: hvJq6bwDAySlGMAzVwBuS, d: 08:40:07, }"] [2025/04/06 08:26:05.109 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 902, b: 902, c: o, ahRhZ: VgauzvAfkJ, d: 08:40:08, }"] [row2="{ b: 902, c: xSoWnF, ahRhZ: 9gjYCofZh0EmG7TrJpW3ImEK65enD3a3LEEeqfcFhEjk, d: 08:40:08, a: 902, }"] [2025/04/06 08:26:05.209 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 903, b: 903, c: o, ahRhZ: VgauzvAfkJ, d: 08:40:09, }"] [row2="{ a: 903, b: 903, c: dRR, ahRhZ: 5F7X3H5MODEQkB8pREMGANQHGn8C1Gi5gV3mnqvuBGor4, d: 08:40:09, }"] [2025/04/06 08:26:05.309 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ d: 08:40:10, a: 904, b: 904, c: o, ahRhZ: VgauzvAfkJ, }"] [row2="{ b: 904, c: l5e7P8Y27P, ahRhZ: VCSpunRi84Yqx, d: 08:40:10, a: 904, }"] [2025/04/06 08:26:05.409 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ d: 08:40:11, a: 905, b: 905, c: o, ahRhZ: VgauzvAfkJ, }"] [row2="{ a: 905, b: 905, c: h2oJFSG, ahRhZ: AO995nmMsuJc3QqS6F8PZr595wo, d: 08:40:11, }"] [2025/04/06 08:26:05.510 +08:00] [WARN] [diff.go:937] ["find different row"] [column=ahRhZ] [row1="{ a: 906, b: 906, c: o, ahRhZ: VgauzvAfkJ, d: 08:40:12, }"] [row2="{ ahRhZ: uITnuYmBIstJVIuJWldNNs9E5HUqLSc9Z, d: 08:40:12, a: 906, b: 906, c: jei7h, }"] [2025/04/06 08:26:05.610 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 907, b: 907, c: o, ahRhZ: VgauzvAfkJ, d: 08:40:13, }"] [row2="{ d: 08:40:13, a: 907, b: 907, c: Tb8UVccMPf, ahRhZ: EK6slYs, }"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2025/04/06 08:26:05.710 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ d: 08:40:14, a: 908, b: 908, c: o, ahRhZ: VgauzvAfkJ, }"] [row2="{ ahRhZ: lKExpq, d: 08:40:14, a: 908, b: 908, c: aPJu0h5L, }"] [2025/04/06 08:26:05.810 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ b: 909, c: o, ahRhZ: VgauzvAfkJ, d: 08:40:15, a: 909, }"] [row2="{ a: 909, b: 909, c: R7, ahRhZ: zGIND, d: 08:40:15, }"] [2025/04/06 08:26:05.910 +08:00] [WARN] [diff.go:937] ["find different row"] [column=ahRhZ] [row1="{ a: 910, b: 910, c: o, ahRhZ: VgauzvAfkJ, d: 08:40:16, }"] [row2="{ b: 910, c: ewm, ahRhZ: u0DKDKsunNWV1tzfrp, d: 08:40:16, a: 910, }"] [2025/04/06 08:26:06.011 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 911, b: 911, c: o, ahRhZ: VgauzvAfkJ, d: 08:40:17, }"] [row2="{ a: 911, b: 911, c: CBrYa, ahRhZ: eTHvGt, d: 08:40:17, }"] [2025/04/06 08:26:06.111 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ b: 912, c: o, ahRhZ: VgauzvAfkJ, d: 08:40:18, a: 912, }"] [row2="{ c: NGzV3T, ahRhZ: gH8T5lGxZyCoc0WjyzxcRoGi3gE, d: 08:40:18, a: 912, b: 912, }"] [2025/04/06 08:26:06.211 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ d: 08:40:19, a: 913, b: 913, c: o, ahRhZ: VgauzvAfkJ, }"] [row2="{ a: 913, b: 913, c: XrCksr, ahRhZ: QeT9tcCboddzMcdRlyDq6lRIQ4XW32P29vTtda, d: 08:40:19, }"] [2025/04/06 08:26:06.311 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ d: 08:40:20, a: 914, b: 914, c: o, ahRhZ: VgauzvAfkJ, }"] [row2="{ a: 914, b: 914, c: v5iQw, ahRhZ: 6oKFYq1yAzvVnfJYMBl3CF3jRv9C02thL0, d: 08:40:20, }"] [2025/04/06 08:26:06.411 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ ahRhZ: VgauzvAfkJ, d: 08:40:21, a: 915, b: 915, c: o, }"] [row2="{ a: 915, b: 915, c: G6zH, ahRhZ: rN4fYgsij6IZ8, d: 08:40:21, }"] + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.savepoint.cli.14878.out cli tso query --pd=http://127.0.0.1:2379 [2025/04/06 08:26:06.512 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ ahRhZ: VgauzvAfkJ, d: 08:40:22, a: 916, b: 916, c: o, }"] [row2="{ d: 08:40:22, a: 916, b: 916, c: Ir40L, ahRhZ: xLX4jviq6tY4wR7wtSEM2Vaa9cAC, }"] [2025/04/06 08:26:06.612 +08:00] [WARN] [diff.go:937] ["find different row"] [column=ahRhZ] [row1="{ b: 917, c: o, ahRhZ: VgauzvAfkJ, d: 08:40:23, a: 917, }"] [row2="{ a: 917, b: 917, c: 9qB, ahRhZ: RFi78iJB9Myz8RMNww7D3cOFGkvv9Hxhc2UpH, d: 08:40:23, }"] [2025/04/06 08:26:06.712 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 918, b: 918, c: o, ahRhZ: VgauzvAfkJ, d: 08:40:24, }"] [row2="{ a: 918, b: 918, c: k, ahRhZ: ImUpXXcbjk, d: 08:40:24, }"] [2025/04/06 08:26:06.812 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ ahRhZ: VgauzvAfkJ, d: 08:40:25, a: 919, b: 919, c: o, }"] [row2="{ a: 919, b: 919, c: AxD, ahRhZ: dA4L, d: 08:40:25, }"] [2025/04/06 08:26:06.912 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ b: 920, c: o, ahRhZ: VgauzvAfkJ, d: 08:40:26, a: 920, }"] [row2="{ a: 920, b: 920, c: 5Z, ahRhZ: uucKUHqIY1jLoLoa3eHijZDjupsCML3pLrP1W95hgNw, d: 08:40:26, }"] =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/http_proxies/run.sh using Sink-Type: mysql... <<================= +++ dirname /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/http_proxies/run.sh ++ cd /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/http_proxies ++ pwd + CUR=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/http_proxies + source /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/http_proxies/../_utils/test_prepare ++ UP_TIDB_HOST=127.0.0.1 ++ UP_TIDB_PORT=4000 ++ UP_TIDB_OTHER_PORT=4001 ++ UP_TIDB_STATUS=10080 ++ UP_TIDB_OTHER_STATUS=10081 ++ DOWN_TIDB_HOST=127.0.0.1 ++ DOWN_TIDB_PORT=3306 ++ DOWN_TIDB_STATUS=20080 ++ TLS_TIDB_HOST=127.0.0.1 ++ TLS_TIDB_PORT=3307 ++ TLS_TIDB_STATUS=30080 ++ UP_PD_HOST_1=127.0.0.1 ++ UP_PD_PORT_1=2379 ++ UP_PD_PEER_PORT_1=2380 ++ UP_PD_HOST_2=127.0.0.1 ++ UP_PD_PORT_2=2679 ++ UP_PD_PEER_PORT_2=2680 ++ UP_PD_HOST_3=127.0.0.1 ++ UP_PD_PORT_3=2779 ++ UP_PD_PEER_PORT_3=2780 ++ DOWN_PD_HOST=127.0.0.1 ++ DOWN_PD_PORT=2479 ++ DOWN_PD_PEER_PORT=2480 ++ TLS_PD_HOST=127.0.0.1 ++ TLS_PD_PORT=2579 ++ TLS_PD_PEER_PORT=2580 ++ UP_TIKV_HOST_1=127.0.0.1 ++ UP_TIKV_PORT_1=20160 ++ UP_TIKV_STATUS_PORT_1=20181 ++ UP_TIKV_HOST_2=127.0.0.1 ++ UP_TIKV_PORT_2=20161 ++ UP_TIKV_STATUS_PORT_2=20182 ++ UP_TIKV_HOST_3=127.0.0.1 ++ UP_TIKV_PORT_3=20162 ++ UP_TIKV_STATUS_PORT_3=20183 ++ DOWN_TIKV_HOST=127.0.0.1 ++ DOWN_TIKV_PORT=21160 ++ DOWN_TIKV_STATUS_PORT=21180 ++ TLS_TIKV_HOST=127.0.0.1 ++ TLS_TIKV_PORT=22160 ++ TLS_TIKV_STATUS_PORT=22180 +++ cat /tmp/tidb_cdc_test/KAFKA_VERSION +++ echo 2.4.1 ++ KAFKA_VERSION=2.4.1 + WORK_DIR=/tmp/tidb_cdc_test/http_proxies + CDC_BINARY=cdc.test + SINK_TYPE=mysql + TEST_HOST_LIST=(UP_TIDB_HOST UP_PD_HOST_{1..3} UP_TIKV_HOST_{1..3}) + lan_addrs=($(hostname -I)) ++ hostname -I + lan_addr=10.233.76.109 + export UP_TIDB_HOST=10.233.76.109 UP_PD_HOST_1=10.233.76.109 UP_PD_HOST_2=10.233.76.109 UP_PD_HOST_3=10.233.76.109 UP_TIKV_HOST_1=10.233.76.109 UP_TIKV_HOST_2=10.233.76.109 UP_TIKV_HOST_3=10.233.76.109 + UP_TIDB_HOST=10.233.76.109 + UP_PD_HOST_1=10.233.76.109 + UP_PD_HOST_2=10.233.76.109 + UP_PD_HOST_3=10.233.76.109 + UP_TIKV_HOST_1=10.233.76.109 + UP_TIKV_HOST_2=10.233.76.109 + UP_TIKV_HOST_3=10.233.76.109 + proxy_pid= ++ shuf -i 10081-20081 -n1 + proxy_port=13444 + trap 'stop_tidb_cluster && stop_proxy' EXIT + '[' mysql == mysql ']' + prepare + for host in '"${TEST_HOST_LIST[@]}"' ++ printenv UP_TIDB_HOST + echo 'UP_TIDB_HOST 10.233.76.109' UP_TIDB_HOST 10.233.76.109 + case $(printenv $host) in ++ printenv UP_TIDB_HOST + for host in '"${TEST_HOST_LIST[@]}"' ++ printenv UP_PD_HOST_1 + echo 'UP_PD_HOST_1 10.233.76.109' UP_PD_HOST_1 10.233.76.109 + case $(printenv $host) in ++ printenv UP_PD_HOST_1 + for host in '"${TEST_HOST_LIST[@]}"' ++ printenv UP_PD_HOST_2 + echo 'UP_PD_HOST_2 10.233.76.109' UP_PD_HOST_2 10.233.76.109 + case $(printenv $host) in ++ printenv UP_PD_HOST_2 + for host in '"${TEST_HOST_LIST[@]}"' ++ printenv UP_PD_HOST_3 + echo 'UP_PD_HOST_3 10.233.76.109' UP_PD_HOST_3 10.233.76.109 + case $(printenv $host) in ++ printenv UP_PD_HOST_3 + for host in '"${TEST_HOST_LIST[@]}"' ++ printenv UP_TIKV_HOST_1 + echo 'UP_TIKV_HOST_1 10.233.76.109' UP_TIKV_HOST_1 10.233.76.109 + case $(printenv $host) in ++ printenv UP_TIKV_HOST_1 + for host in '"${TEST_HOST_LIST[@]}"' ++ printenv UP_TIKV_HOST_2 + echo 'UP_TIKV_HOST_2 10.233.76.109' UP_TIKV_HOST_2 10.233.76.109 + case $(printenv $host) in ++ printenv UP_TIKV_HOST_2 + for host in '"${TEST_HOST_LIST[@]}"' ++ printenv UP_TIKV_HOST_3 + echo 'UP_TIKV_HOST_3 10.233.76.109' UP_TIKV_HOST_3 10.233.76.109 + case $(printenv $host) in ++ printenv UP_TIKV_HOST_3 + rm -rf /tmp/tidb_cdc_test/http_proxies + mkdir -p /tmp/tidb_cdc_test/http_proxies + stop_tidb_cluster [2025/04/06 08:26:07.013 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ b: 921, c: o, ahRhZ: VgauzvAfkJ, d: 08:40:27, a: 921, }"] [row2="{ a: 921, b: 921, c: m, ahRhZ: PTKqJwwP7inEX9dPnNoTy1IPDtbVOB7UzaAB, d: 08:40:27, }"] [2025/04/06 08:26:07.113 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ c: o, ahRhZ: VgauzvAfkJ, d: 08:40:28, a: 922, b: 922, }"] [row2="{ a: 922, b: 922, c: opa, ahRhZ: 3k, d: 08:40:28, }"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 06 Apr 2025 00:26:06 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3ec5eb26-a263-426a-8000-fde98f3ca908 {"id":"3ec5eb26-a263-426a-8000-fde98f3ca908","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899164} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087d4b16e3 3ec5eb26-a263-426a-8000-fde98f3ca908 /tidb/cdc/default/default/upstream/7489989827991870654 {"id":7489989827991870654,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3ec5eb26-a263-426a-8000-fde98f3ca908 {"id":"3ec5eb26-a263-426a-8000-fde98f3ca908","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899164} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087d4b16e3 3ec5eb26-a263-426a-8000-fde98f3ca908 /tidb/cdc/default/default/upstream/7489989827991870654 {"id":7489989827991870654,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3ec5eb26-a263-426a-8000-fde98f3ca908 {"id":"3ec5eb26-a263-426a-8000-fde98f3ca908","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899164} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087d4b16e3 3ec5eb26-a263-426a-8000-fde98f3ca908 /tidb/cdc/default/default/upstream/7489989827991870654 {"id":7489989827991870654,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ci_collation_compatibility.cli.33613.out cli changefeed create --start-ts=457152702049943553 '--sink-uri=mysql://normal:123456@127.0.0.1:3306/?safe-mode=true' [2025/04/06 08:26:07.213 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 923, b: 923, c: o, ahRhZ: VgauzvAfkJ, d: 08:40:29, }"] [row2="{ c: X6, ahRhZ: YLV8zeWF6QtR20GVU8boh0lVo5GQIvTEE3L6hxZXiS, d: 08:40:29, a: 923, b: 923, }"] [2025/04/06 08:26:07.313 +08:00] [WARN] [diff.go:937] ["find different row"] [column=ahRhZ] [row1="{ ahRhZ: VgauzvAfkJ, d: 08:40:30, a: 924, b: 924, c: o, }"] [row2="{ a: 924, b: 924, c: 2Rl3mCKie, ahRhZ: OwSX01FrqsMhWEPnTvZ4WWgZoj7e2AihwwsUJ6r, d: 08:40:30, }"] [2025/04/06 08:26:07.413 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 925, b: 925, c: o, ahRhZ: VgauzvAfkJ, d: 08:40:31, }"] [row2="{ d: 08:40:31, a: 925, b: 925, c: 6, ahRhZ: 2kV, }"] Create changefeed successfully! ID: dea03d6d-2d28-4fa5-8d15-491d772483c1 Info: {"upstream_id":7489989827991870654,"namespace":"default","id":"dea03d6d-2d28-4fa5-8d15-491d772483c1","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/?safe-mode=true","create_time":"2025-04-06T08:26:07.386771466+08:00","start_ts":457152702049943553,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457152702049943553,"checkpoint_ts":457152702049943553,"checkpoint_time":"2025-04-06 08:26:02.483"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... [2025/04/06 08:26:07.514 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 926, b: 926, c: o, ahRhZ: VgauzvAfkJ, d: 08:40:32, }"] [row2="{ ahRhZ: m8zXX7AmM82ImMcYgNLGg7pAGZwJlb30KFpAI8, d: 08:40:32, a: 926, b: 926, c: h2JzWErEKC, }"] [2025/04/06 08:26:07.614 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 927, b: 927, c: o, ahRhZ: VgauzvAfkJ, d: 08:40:33, }"] [row2="{ a: 927, b: 927, c: ZhL, ahRhZ: YxpDhHIxCOb, d: 08:40:33, }"] table owner_remove_table_error.finished_mark not exists for 1-th check, retry later [2025/04/06 08:26:07.714 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 928, b: 928, c: o, ahRhZ: VgauzvAfkJ, d: 08:40:34, }"] [row2="{ d: 08:40:34, a: 928, b: 928, c: fp, ahRhZ: YoISupHq3TUrLn8YPlszNnfz5MBYS3a, }"] [2025/04/06 08:26:07.814 +08:00] [WARN] [diff.go:937] ["find different row"] [column=c] [row1="{ a: 929, b: 929, c: o, ahRhZ: VgauzvAfkJ, d: 08:40:35, }"] [row2="{ a: 929, b: 929, c: avwYxvtczC, ahRhZ: m1fdMjzpfg5bjqtA2dDKFsDQ2, d: 08:40:35, }"] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489989862946304354 TiDB Cluster ID. tikv_gc_leader_uuid 65821f60f28000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-8-31x9h, pid:38718, start at 2025-04-06 08:26:05.683197812 +0800 CST m=+1.989411678 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:28:05.690 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:26:05.692 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:16:05.692 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489989862946304354 TiDB Cluster ID. tikv_gc_leader_uuid 65821f60f28000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-8-31x9h, pid:38718, start at 2025-04-06 08:26:05.683197812 +0800 CST m=+1.989411678 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:28:05.690 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:26:05.692 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:16:05.692 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489989864820129251 TiDB Cluster ID. tikv_gc_leader_uuid 65821f61238000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-8-31x9h, pid:38798, start at 2025-04-06 08:26:05.877354026 +0800 CST m=+2.130692940 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:28:05.884 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:26:05.923 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:16:05.923 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/http_api_tls_with_user_auth/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/http_api_tls_with_user_auth/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/http_api_tls_with_user_auth/tiflash/db/proxy"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/http_api_tls_with_user_auth/tiflash/log/proxy.log"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/http_api_tls_with_user_auth/tiflash-proxy.toml"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + set +x + tso='457152703192891394 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 457152703192891394 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Sun Apr 6 08:26:08 CST 2025] <<<<<< START cdc server in savepoint case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.savepoint.1491914921.out server --log-file /tmp/tidb_cdc_test/savepoint/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/savepoint/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 start tidb cluster in /tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... + set +x table owner_remove_table_error.finished_mark not exists for 2-th check, retry later The 1 times to try to start tls tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/http_api_tls_with_user_auth Starting TLS PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 + start_tidb_cluster --workdir /tmp/tidb_cdc_test/http_proxies The 1 times to try to start tidb cluster... [2025/04/06 08:26:08.891 +08:00] [INFO] [main.go:61] ["table mover started"] [2025/04/06 08:26:08.893 +08:00] [INFO] [main.go:165] ["new cluster initialized"] [2025/04/06 08:26:08.894 +08:00] [DEBUG] [main.go:191] ["retrieved owner ID"] [ownerID=6f229d06-016c-4169-bab4-9209ad4184b8] [2025/04/06 08:26:08.894 +08:00] [DEBUG] [main.go:198] ["retrieved owner addr"] [ownerAddr=127.0.0.1:8300] [2025/04/06 08:26:08.894 +08:00] [DEBUG] [main.go:209] ["retrieved changefeeds"] [changefeedsError="json: unsupported type: map[model.ChangeFeedID]*mvccpb.KeyValue"] [2025/04/06 08:26:09.033 +08:00] [DEBUG] [main.go:228] ["retrieved processor details"] [changefeed=44202c40-4a17-4681-8452-63490f9b5e23] [captureID=33241d46-9f8f-4c49-9223-9ee69e44e36f] [processorDetail="{\"table_ids\":[]}"] [2025/04/06 08:26:09.183 +08:00] [DEBUG] [main.go:228] ["retrieved processor details"] [changefeed=44202c40-4a17-4681-8452-63490f9b5e23] [captureID=4322f23c-3086-4cf3-a91d-034ceb577f00] [processorDetail="{\"table_ids\":[]}"] [2025/04/06 08:26:09.333 +08:00] [DEBUG] [main.go:228] ["retrieved processor details"] [changefeed=44202c40-4a17-4681-8452-63490f9b5e23] [captureID=6f229d06-016c-4169-bab4-9209ad4184b8] [processorDetail="{\"table_ids\":[114,116]}"] [2025/04/06 08:26:09.333 +08:00] [INFO] [main.go:75] ["task status"] [status="{\"33241d46-9f8f-4c49-9223-9ee69e44e36f\":[],\"4322f23c-3086-4cf3-a91d-034ceb577f00\":[],\"6f229d06-016c-4169-bab4-9209ad4184b8\":[{\"ID\":114,\"Changefeed\":\"44202c40-4a17-4681-8452-63490f9b5e23\"},{\"ID\":116,\"Changefeed\":\"44202c40-4a17-4681-8452-63490f9b5e23\"}]}"] [2025/04/06 08:26:09.333 +08:00] [DEBUG] [main.go:287] ["preparing HTTP API call to owner"] [formStr="cf-id=44202c40-4a17-4681-8452-63490f9b5e23&target-cp-id=4322f23c-3086-4cf3-a91d-034ceb577f00&table-id=114"] [2025/04/06 08:26:09.383 +08:00] [INFO] [main.go:179] ["moved table successful"] [tableID=114] [2025/04/06 08:26:09.383 +08:00] [DEBUG] [main.go:287] ["preparing HTTP API call to owner"] [formStr="cf-id=44202c40-4a17-4681-8452-63490f9b5e23&target-cp-id=4322f23c-3086-4cf3-a91d-034ceb577f00&table-id=116"] [2025/04/06 08:26:09.433 +08:00] [INFO] [main.go:179] ["moved table successful"] [tableID=116] [2025/04/06 08:26:09.433 +08:00] [INFO] [main.go:113] ["all tables are moved"] [sourceCapture=6f229d06-016c-4169-bab4-9209ad4184b8] [targetCapture=4322f23c-3086-4cf3-a91d-034ceb577f00] table move_table.check1 exists check diff successfully + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 06 Apr 2025 00:26:11 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f5026406-8f9e-40cc-95e2-133d5d54f2df {"id":"f5026406-8f9e-40cc-95e2-133d5d54f2df","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899168} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087d5837e4 f5026406-8f9e-40cc-95e2-133d5d54f2df /tidb/cdc/default/default/upstream/7489989843854196779 {"id":7489989843854196779,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f5026406-8f9e-40cc-95e2-133d5d54f2df {"id":"f5026406-8f9e-40cc-95e2-133d5d54f2df","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899168} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087d5837e4 f5026406-8f9e-40cc-95e2-133d5d54f2df /tidb/cdc/default/default/upstream/7489989843854196779 {"id":7489989843854196779,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f5026406-8f9e-40cc-95e2-133d5d54f2df {"id":"f5026406-8f9e-40cc-95e2-133d5d54f2df","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899168} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087d5837e4 f5026406-8f9e-40cc-95e2-133d5d54f2df /tidb/cdc/default/default/upstream/7489989843854196779 {"id":7489989843854196779,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.savepoint.cli.14981.out cli changefeed create --start-ts=457152703192891394 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ table owner_remove_table_error.finished_mark exists check diff successfully \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } [Pipeline] // timeout [Pipeline] } Create changefeed successfully! ID: 4e656c06-4ff6-4056-8ebf-b5156329cfc9 Info: {"upstream_id":7489989843854196779,"namespace":"default","id":"4e656c06-4ff6-4056-8ebf-b5156329cfc9","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-06T08:26:11.835911384+08:00","start_ts":457152703192891394,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457152703192891394,"checkpoint_ts":457152703192891394,"checkpoint_time":"2025-04-06 08:26:06.843"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... Cache not saved (ws/jenkins-pingcap-tiflow-periodics_cdc_integration_mysql_test-234/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release [Pipeline] // dir [Pipeline] } [Pipeline] // stage wait process cdc.test exit for 1-th time... [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } wait process cdc.test exit for 2-th time... + set +x wait process cdc.test exit for 3-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit ____________________________________ *************************** 1. row *************************** primary_ts: 457152685670400000 secondary_ts: 457152687261876264 primary_ts is 457152685670400000 start_ts is 457152685584678913 check successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Sun Apr 6 08:26:07 CST 2025] <<<<<< run test case syncpoint_check_ts success! >>>>>> [Sun Apr 6 08:26:13 CST 2025] <<<<<< run test case owner_remove_table_error success! >>>>>> Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting TLS TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release [2025/04/06 08:26:14.183 +08:00] [INFO] [main.go:61] ["table mover started"] [2025/04/06 08:26:14.249 +08:00] [INFO] [main.go:165] ["new cluster initialized"] [2025/04/06 08:26:14.250 +08:00] [DEBUG] [main.go:191] ["retrieved owner ID"] [ownerID=6f229d06-016c-4169-bab4-9209ad4184b8] [2025/04/06 08:26:14.250 +08:00] [DEBUG] [main.go:198] ["retrieved owner addr"] [ownerAddr=127.0.0.1:8300] [2025/04/06 08:26:14.251 +08:00] [DEBUG] [main.go:209] ["retrieved changefeeds"] [changefeedsError="json: unsupported type: map[model.ChangeFeedID]*mvccpb.KeyValue"] [2025/04/06 08:26:14.333 +08:00] [DEBUG] [main.go:228] ["retrieved processor details"] [changefeed=44202c40-4a17-4681-8452-63490f9b5e23] [captureID=33241d46-9f8f-4c49-9223-9ee69e44e36f] [processorDetail="{\"table_ids\":[]}"] [2025/04/06 08:26:14.633 +08:00] [DEBUG] [main.go:228] ["retrieved processor details"] [changefeed=44202c40-4a17-4681-8452-63490f9b5e23] [captureID=4322f23c-3086-4cf3-a91d-034ceb577f00] [processorDetail="{\"table_ids\":[116]}"] [2025/04/06 08:26:14.783 +08:00] [DEBUG] [main.go:228] ["retrieved processor details"] [changefeed=44202c40-4a17-4681-8452-63490f9b5e23] [captureID=6f229d06-016c-4169-bab4-9209ad4184b8] [processorDetail="{\"table_ids\":[118]}"] [2025/04/06 08:26:14.783 +08:00] [INFO] [main.go:75] ["task status"] [status="{\"33241d46-9f8f-4c49-9223-9ee69e44e36f\":[],\"4322f23c-3086-4cf3-a91d-034ceb577f00\":[{\"ID\":116,\"Changefeed\":\"44202c40-4a17-4681-8452-63490f9b5e23\"}],\"6f229d06-016c-4169-bab4-9209ad4184b8\":[{\"ID\":118,\"Changefeed\":\"44202c40-4a17-4681-8452-63490f9b5e23\"}]}"] [2025/04/06 08:26:14.783 +08:00] [DEBUG] [main.go:287] ["preparing HTTP API call to owner"] [formStr="cf-id=44202c40-4a17-4681-8452-63490f9b5e23&target-cp-id=6f229d06-016c-4169-bab4-9209ad4184b8&table-id=116"] [2025/04/06 08:26:14.833 +08:00] [INFO] [main.go:179] ["moved table successful"] [tableID=116] [2025/04/06 08:26:14.833 +08:00] [INFO] [main.go:113] ["all tables are moved"] [sourceCapture=4322f23c-3086-4cf3-a91d-034ceb577f00] [targetCapture=6f229d06-016c-4169-bab4-9209ad4184b8] table savepoint.finish_mark not exists for 1-th check, retry later check diff successfully table move_table.check2 not exists for 1-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/http_proxies Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... [2025/04/06 08:26:14.930 +08:00] [WARN] [diff.go:766] ["rows is not equal"] [table=`test`.`itest`] [where="((TRUE) AND TRUE)"] [cost=1m31.665237277s] [2025/04/06 08:26:14.976 +08:00] [WARN] [diff.go:384] ["check chunk data not equal"] [chunk="{\"id\":0,\"bounds\":[],\"where\":\"((TRUE) AND TRUE)\",\"args\":null,\"state\":\"failed\"}"] Starting TLS TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying TLS TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table savepoint.finish_mark exists check diff successfully table move_table.check2 not exists for 2-th check, retry later wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489989914123752837 TiDB Cluster ID. tikv_gc_leader_uuid 65821f6c5540004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-8-31x9h, pid:40364, start at 2025-04-06 08:26:17.306660307 +0800 CST m=+1.354026559 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:28:17.313 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:26:17.301 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:16:17.301 +0800 All versions after safe point can be accessed. (DO NOT EDIT) [Sun Apr 6 08:26:18 CST 2025] <<<<<< START cdc server in http_api_tls_with_user_auth case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates ']' + curl_status_cmd='curl --cacert /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client.pem --key /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client-key.pem --user ticdc:ticdc_secret -vsL --max-time 20 https://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.http_api_tls_with_user_auth.4042040422.out server --log-file /tmp/tidb_cdc_test/http_api_tls_with_user_auth/cdc_http_api_tls_with_user_auth_tls1.log --log-level debug --data-dir /tmp/tidb_cdc_test/http_api_tls_with_user_auth/cdc_data_http_api_tls_with_user_auth_tls1 --cluster-id default --config /tmp/tidb_cdc_test/http_api_tls_with_user_auth/server.toml --ca /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/server.pem --key /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/server-key.pem --cert-allowed-cn client --addr 127.0.0.1:8300 --pd https://127.0.0.1:2579 + (( i <= 50 )) ++ curl --cacert /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client.pem --key /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client-key.pem --user ticdc:ticdc_secret -vsL --max-time 20 https://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489989903191531286 TiDB Cluster ID. tikv_gc_leader_uuid 65821f6b174000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-6-zhp59, pid:12920, start at 2025-04-06 08:26:16.041059956 +0800 CST m=+2.261311414 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:28:16.047 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:26:16.029 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:16:16.029 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489989903191531286 TiDB Cluster ID. tikv_gc_leader_uuid 65821f6b174000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-6-zhp59, pid:12920, start at 2025-04-06 08:26:16.041059956 +0800 CST m=+2.261311414 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:28:16.047 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:26:16.029 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:16:16.029 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489989910755179864 TiDB Cluster ID. tikv_gc_leader_uuid 65821f6b3bc000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-6-zhp59, pid:12996, start at 2025-04-06 08:26:16.215801357 +0800 CST m=+2.361777079 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:28:16.238 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:26:16.229 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:16:16.229 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/tiflash/log/error.log arg matches is ArgMatches { args: {"addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/tiflash/log/proxy.log"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Sun Apr 6 08:26:18 CST 2025] <<<<<< run test case savepoint success! >>>>>> =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/server_config_compatibility/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table move_table.check2 exists check diff successfully wait process cdc.test exit for 1-th time... [Sun Apr 6 08:26:20 CST 2025] <<<<<< START cdc server in consistent_replicate_storage_file_large_value case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.consistent_replicate_storage_file_large_value.1426614268.out server --log-file /tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 wait process cdc.test exit for 2-th time... Starting Upstream TiDB... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Sun Apr 6 08:26:20 CST 2025] <<<<<< run test case move_table success! >>>>>> Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '10.233.76.109' (111) [2025/04/06 08:26:20.581 +08:00] [WARN] [diff.go:557] ["checksum is not equal"] [table=`test`.`ntest`] [where="((TRUE) AND TRUE)"] [sourceChecksum=0] [targetChecksum=3236231632] [getSourceChecksumCost=1.72482ms] [getTargetChecksumCost=8.326121ms] [2025/04/06 08:26:06.590 +08:00] [INFO] [main.go:86] ["running ddl test: 0 createDropSchemaDDL"] [2025/04/06 08:26:06.884 +08:00] [INFO] [main.go:221] ["1 insert success: 100"] [2025/04/06 08:26:06.889 +08:00] [INFO] [main.go:221] ["0 insert success: 100"] [2025/04/06 08:26:07.151 +08:00] [INFO] [main.go:221] ["1 insert success: 200"] [2025/04/06 08:26:07.154 +08:00] [INFO] [main.go:235] ["1 delete success: 100"] [2025/04/06 08:26:07.157 +08:00] [INFO] [main.go:221] ["0 insert success: 200"] [2025/04/06 08:26:07.160 +08:00] [INFO] [main.go:235] ["0 delete success: 100"] [2025/04/06 08:26:08.471 +08:00] [INFO] [main.go:221] ["1 insert success: 300"] [2025/04/06 08:26:08.475 +08:00] [INFO] [main.go:221] ["0 insert success: 300"] [2025/04/06 08:26:08.761 +08:00] [INFO] [main.go:221] ["0 insert success: 400"] [2025/04/06 08:26:08.764 +08:00] [INFO] [main.go:235] ["0 delete success: 200"] [2025/04/06 08:26:08.766 +08:00] [INFO] [main.go:221] ["1 insert success: 400"] [2025/04/06 08:26:08.770 +08:00] [INFO] [main.go:235] ["1 delete success: 200"] [2025/04/06 08:26:10.054 +08:00] [INFO] [main.go:221] ["0 insert success: 500"] [2025/04/06 08:26:10.054 +08:00] [INFO] [main.go:221] ["1 insert success: 500"] [2025/04/06 08:26:10.316 +08:00] [INFO] [main.go:221] ["1 insert success: 600"] [2025/04/06 08:26:10.326 +08:00] [INFO] [main.go:235] ["1 delete success: 300"] [2025/04/06 08:26:10.328 +08:00] [INFO] [main.go:221] ["0 insert success: 600"] [2025/04/06 08:26:10.337 +08:00] [INFO] [main.go:235] ["0 delete success: 300"] [2025/04/06 08:26:17.104 +08:00] [INFO] [main.go:86] ["running ddl test: 1 truncateDDL"] [2025/04/06 08:26:17.387 +08:00] [INFO] [main.go:221] ["0 insert success: 100"] [2025/04/06 08:26:17.389 +08:00] [INFO] [main.go:221] ["1 insert success: 100"] [2025/04/06 08:26:17.634 +08:00] [INFO] [main.go:221] ["1 insert success: 200"] [2025/04/06 08:26:17.639 +08:00] [INFO] [main.go:235] ["0 delete success: 100"] [2025/04/06 08:26:17.640 +08:00] [INFO] [main.go:235] ["1 delete success: 100"] [2025/04/06 08:26:17.640 +08:00] [INFO] [main.go:221] ["0 insert success: 200"] [2025/04/06 08:26:17.867 +08:00] [INFO] [main.go:221] ["1 insert success: 300"] [2025/04/06 08:26:17.875 +08:00] [INFO] [main.go:221] ["0 insert success: 300"] [2025/04/06 08:26:18.103 +08:00] [INFO] [main.go:221] ["1 insert success: 400"] [2025/04/06 08:26:18.107 +08:00] [INFO] [main.go:235] ["0 delete success: 200"] [2025/04/06 08:26:18.108 +08:00] [INFO] [main.go:221] ["0 insert success: 400"] [2025/04/06 08:26:18.109 +08:00] [INFO] [main.go:235] ["1 delete success: 200"] [2025/04/06 08:26:18.334 +08:00] [INFO] [main.go:221] ["1 insert success: 500"] [2025/04/06 08:26:18.347 +08:00] [INFO] [main.go:221] ["0 insert success: 500"] [2025/04/06 08:26:18.583 +08:00] [INFO] [main.go:221] ["1 insert success: 600"] [2025/04/06 08:26:18.590 +08:00] [INFO] [main.go:235] ["1 delete success: 300"] [2025/04/06 08:26:18.596 +08:00] [INFO] [main.go:221] ["0 insert success: 600"] [2025/04/06 08:26:18.599 +08:00] [INFO] [main.go:235] ["0 delete success: 300"] [2025/04/06 08:26:18.823 +08:00] [INFO] [main.go:221] ["1 insert success: 700"] [2025/04/06 08:26:18.824 +08:00] [INFO] [main.go:221] ["0 insert success: 700"] [2025/04/06 08:26:19.065 +08:00] [INFO] [main.go:221] ["1 insert success: 800"] [2025/04/06 08:26:19.069 +08:00] [INFO] [main.go:221] ["0 insert success: 800"] [2025/04/06 08:26:19.072 +08:00] [INFO] [main.go:235] ["0 delete success: 400"] [2025/04/06 08:26:19.073 +08:00] [INFO] [main.go:235] ["1 delete success: 400"] [2025/04/06 08:26:19.313 +08:00] [INFO] [main.go:221] ["0 insert success: 900"] [2025/04/06 08:26:19.314 +08:00] [INFO] [main.go:221] ["1 insert success: 900"] [2025/04/06 08:26:19.548 +08:00] [INFO] [main.go:221] ["1 insert success: 1000"] [2025/04/06 08:26:19.553 +08:00] [INFO] [main.go:221] ["0 insert success: 1000"] [2025/04/06 08:26:19.562 +08:00] [INFO] [main.go:235] ["1 delete success: 500"] [2025/04/06 08:26:19.562 +08:00] [INFO] [main.go:235] ["0 delete success: 500"] [2025/04/06 08:26:19.808 +08:00] [INFO] [main.go:221] ["1 insert success: 1100"] [2025/04/06 08:26:19.812 +08:00] [INFO] [main.go:221] ["0 insert success: 1100"] [2025/04/06 08:26:20.081 +08:00] [INFO] [main.go:221] ["1 insert success: 1200"] [2025/04/06 08:26:20.086 +08:00] [INFO] [main.go:221] ["0 insert success: 1200"] [2025/04/06 08:26:20.096 +08:00] [INFO] [main.go:235] ["1 delete success: 600"] [2025/04/06 08:26:20.096 +08:00] [INFO] [main.go:235] ["0 delete success: 600"] [2025/04/06 08:26:20.377 +08:00] [INFO] [main.go:221] ["1 insert success: 1300"] [2025/04/06 08:26:20.380 +08:00] [INFO] [main.go:221] ["0 insert success: 1300"] [2025/04/06 08:26:20.636 +08:00] [INFO] [main.go:221] ["1 insert success: 1400"] [2025/04/06 08:26:20.641 +08:00] [INFO] [main.go:221] ["0 insert success: 1400"] [2025/04/06 08:26:20.647 +08:00] [INFO] [main.go:235] ["1 delete success: 700"] [2025/04/06 08:26:20.650 +08:00] [INFO] [main.go:235] ["0 delete success: 700"] [2025/04/06 08:26:20.935 +08:00] [INFO] [main.go:221] ["1 insert success: 1500"] [2025/04/06 08:26:20.946 +08:00] [INFO] [main.go:221] ["0 insert success: 1500"] [2025/04/06 08:26:21.266 +08:00] [INFO] [main.go:221] ["1 insert success: 1600"] [2025/04/06 08:26:21.277 +08:00] [INFO] [main.go:221] ["0 insert success: 1600"] [2025/04/06 08:26:21.280 +08:00] [INFO] [main.go:235] ["1 delete success: 800"] [2025/04/06 08:26:21.292 +08:00] [INFO] [main.go:235] ["0 delete success: 800"] + (( i++ )) + (( i <= 50 )) ++ curl --cacert /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client.pem --key /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client-key.pem --user ticdc:ticdc_secret -vsL --max-time 20 https://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Initializing NSS with certpath: sql:/etc/pki/nssdb * CAfile: /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem CApath: none * NSS: client certificate from file * subject: CN=client * start date: Feb 18 07:48:00 2020 GMT * expire date: Jan 25 07:48:00 2120 GMT * common name: client * issuer: CN=My own CA,O=PingCAP,L=Beijing,ST=Beijing,C=CN * SSL connection using TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 * Server certificate: * subject: CN=tidb-server * start date: Feb 18 09:11:00 2020 GMT * expire date: Jan 25 09:11:00 2120 GMT * common name: tidb-server * issuer: CN=My own CA,O=PingCAP,L=Beijing,ST=Beijing,C=CN * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 06 Apr 2025 00:26:21 GMT < Content-Length: 1253 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/890e253d-8a87-438d-afc6-63906c93969f {"id":"890e253d-8a87-438d-afc6-63906c93969f","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899178} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/1f5d96087d955add 890e253d-8a87-438d-afc6-63906c93969f /tidb/cdc/default/default/upstream/7489989914123752837 {"id":7489989914123752837,"pd-endpoints":"https://127.0.0.1:2579,https://127.0.0.1:2579","key-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/server-key.pem","cert-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/server.pem","ca-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem","cert-allowed-cn":["client","tidb-server"]}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/890e253d-8a87-438d-afc6-63906c93969f {"id":"890e253d-8a87-438d-afc6-63906c93969f","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899178} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/1f5d96087d955add 890e253d-8a87-438d-afc6-63906c93969f /tidb/cdc/default/default/upstream/7489989914123752837 {"id":7489989914123752837,"pd-endpoints":"https://127.0.0.1:2579,https://127.0.0.1:2579","key-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/server-key.pem","cert-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/server.pem","ca-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem","cert-allowed-cn":["client","tidb-server"]}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/890e253d-8a87-438d-afc6-63906c93969f {"id":"890e253d-8a87-438d-afc6-63906c93969f","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899178} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/1f5d96087d955add 890e253d-8a87-438d-afc6-63906c93969f /tidb/cdc/default/default/upstream/7489989914123752837 {"id":7489989914123752837,"pd-endpoints":"https://127.0.0.1:2579,https://127.0.0.1:2579","key-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/server-key.pem","cert-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/server.pem","ca-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem","cert-allowed-cn":["client","tidb-server"]}' + grep -q 'etcd info' + break + set +x [2025/04/06 08:26:21.527 +08:00] [INFO] [main.go:221] ["1 insert success: 1700"] [2025/04/06 08:26:21.536 +08:00] [INFO] [main.go:221] ["0 insert success: 1700"] [2025/04/06 08:26:21.760 +08:00] [INFO] [main.go:221] ["1 insert success: 1800"] [2025/04/06 08:26:21.764 +08:00] [INFO] [main.go:221] ["0 insert success: 1800"] [2025/04/06 08:26:21.772 +08:00] [INFO] [main.go:235] ["1 delete success: 900"] [2025/04/06 08:26:21.777 +08:00] [INFO] [main.go:235] ["0 delete success: 900"] [2025/04/06 08:26:21.987 +08:00] [INFO] [main.go:221] ["1 insert success: 1900"] [2025/04/06 08:26:22.005 +08:00] [INFO] [main.go:221] ["0 insert success: 1900"] ERROR 2003 (HY000): Can't connect to MySQL server on '10.233.76.109' (111) start tidb cluster in /tmp/tidb_cdc_test/server_config_compatibility Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... [Sun Apr 6 08:26:23 CST 2025] <<<<<< START cdc server in http_api_tls_with_user_auth case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates ']' + curl_status_cmd='curl --cacert /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client.pem --key /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client-key.pem --user ticdc:ticdc_secret -vsL --max-time 20 https://127.0.0.1:8301/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.http_api_tls_with_user_auth.4049240494.out server --log-file /tmp/tidb_cdc_test/http_api_tls_with_user_auth/cdc_http_api_tls_with_user_auth_tls2.log --log-level debug --data-dir /tmp/tidb_cdc_test/http_api_tls_with_user_auth/cdc_data_http_api_tls_with_user_auth_tls2 --cluster-id default --config /tmp/tidb_cdc_test/http_api_tls_with_user_auth/server.toml --ca /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/server.pem --key /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/server-key.pem --cert-allowed-cn client --addr 127.0.0.1:8301 --pd https://127.0.0.1:2579 + (( i = 0 )) + (( i <= 50 )) ++ curl --cacert /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client.pem --key /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client-key.pem --user ticdc:ticdc_secret -vsL --max-time 20 https://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 06 Apr 2025 00:26:23 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d949d1e0-e09c-4817-800c-ba88fb3d3944 {"id":"d949d1e0-e09c-4817-800c-ba88fb3d3944","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899180} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087d8dacd5 d949d1e0-e09c-4817-800c-ba88fb3d3944 /tidb/cdc/default/default/upstream/7489989903191531286 {"id":7489989903191531286,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d949d1e0-e09c-4817-800c-ba88fb3d3944 {"id":"d949d1e0-e09c-4817-800c-ba88fb3d3944","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899180} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087d8dacd5 d949d1e0-e09c-4817-800c-ba88fb3d3944 /tidb/cdc/default/default/upstream/7489989903191531286 {"id":7489989903191531286,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d949d1e0-e09c-4817-800c-ba88fb3d3944 {"id":"d949d1e0-e09c-4817-800c-ba88fb3d3944","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899180} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087d8dacd5 d949d1e0-e09c-4817-800c-ba88fb3d3944 /tidb/cdc/default/default/upstream/7489989903191531286 {"id":7489989903191531286,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x ***************** properties ***************** "fieldlength"="15000" "scanproportion"="0" "readallfields"="true" "mysql.host"="127.0.0.1" "threadcount"="10" "recordcount"="100" "operationcount"="0" "readproportion"="0" "insertproportion"="0" "fieldlengthdistribution"="constant" "mysql.user"="root" "mysql.db"="consistent_replicate_storage_file_large_value" "requestdistribution"="uniform" "updateproportion"="0" "fieldcount"="1000" "mysql.port"="4000" "dotransactions"="false" "workload"="core" ********************************************** VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489989926198349211 TiDB Cluster ID. tikv_gc_leader_uuid 65821f720a8003b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-p-5tjc6, pid:8002, start at 2025-04-06 08:26:23.170692518 +0800 CST m=+2.320851444 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:28:23.177 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:26:23.226 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:16:23.226 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489989926198349211 TiDB Cluster ID. tikv_gc_leader_uuid 65821f720a8003b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-p-5tjc6, pid:8002, start at 2025-04-06 08:26:23.170692518 +0800 CST m=+2.320851444 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:28:23.177 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:26:23.226 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:16:23.226 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489989934147539576 TiDB Cluster ID. tikv_gc_leader_uuid 65821f72188000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-p-5tjc6, pid:8075, start at 2025-04-06 08:26:23.241770173 +0800 CST m=+2.305585088 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:28:23.249 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:26:23.251 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:16:23.251 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/http_proxies/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/http_proxies/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/http_proxies/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["10.233.76.109:2379"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/http_proxies/tiflash/db/proxy"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/http_proxies/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Verifying downstream PD is started... \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } [Pipeline] // timeout [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-periodics_cdc_integration_mysql_test-234/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // stage [Pipeline] } Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release [Pipeline] // container [Pipeline] } [Pipeline] // withEnv + (( i++ )) + (( i <= 50 )) ++ curl --cacert /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client.pem --key /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client-key.pem --user ticdc:ticdc_secret -vsL --max-time 20 https://127.0.0.1:8301/debug/info * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Initializing NSS with certpath: sql:/etc/pki/nssdb * CAfile: /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem CApath: none * NSS: client certificate from file * subject: CN=client * start date: Feb 18 07:48:00 2020 GMT * expire date: Jan 25 07:48:00 2120 GMT * common name: client * issuer: CN=My own CA,O=PingCAP,L=Beijing,ST=Beijing,C=CN * SSL connection using TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 * Server certificate: * subject: CN=tidb-server * start date: Feb 18 09:11:00 2020 GMT * expire date: Jan 25 09:11:00 2120 GMT * common name: tidb-server * issuer: CN=My own CA,O=PingCAP,L=Beijing,ST=Beijing,C=CN * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 06 Apr 2025 00:26:26 GMT < Content-Length: 1714 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1959bee5-3b2f-41f2-90a9-3dc06476a405 {"id":"1959bee5-3b2f-41f2-90a9-3dc06476a405","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899183} /tidb/cdc/default/__cdc_meta__/capture/890e253d-8a87-438d-afc6-63906c93969f {"id":"890e253d-8a87-438d-afc6-63906c93969f","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899178} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/1f5d96087d955add 890e253d-8a87-438d-afc6-63906c93969f /tidb/cdc/default/__cdc_meta__/owner/1f5d96087d955b1d 1959bee5-3b2f-41f2-90a9-3dc06476a405 /tidb/cdc/default/default/upstream/7489989914123752837 {"id":7489989914123752837,"pd-endpoints":"https://127.0.0.1:2579,https://127.0.0.1:2579","key-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/server-key.pem","cert-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/server.pem","ca-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem","cert-allowed-cn":["client","tidb-server"]}' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1959bee5-3b2f-41f2-90a9-3dc06476a405 {"id":"1959bee5-3b2f-41f2-90a9-3dc06476a405","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899183} /tidb/cdc/default/__cdc_meta__/capture/890e253d-8a87-438d-afc6-63906c93969f {"id":"890e253d-8a87-438d-afc6-63906c93969f","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899178} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/1f5d96087d955add 890e253d-8a87-438d-afc6-63906c93969f /tidb/cdc/default/__cdc_meta__/owner/1f5d96087d955b1d 1959bee5-3b2f-41f2-90a9-3dc06476a405 /tidb/cdc/default/default/upstream/7489989914123752837 {"id":7489989914123752837,"pd-endpoints":"https://127.0.0.1:2579,https://127.0.0.1:2579","key-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/server-key.pem","cert-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/server.pem","ca-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem","cert-allowed-cn":["client","tidb-server"]}' + grep -q 'failed to get info:' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1959bee5-3b2f-41f2-90a9-3dc06476a405 {"id":"1959bee5-3b2f-41f2-90a9-3dc06476a405","address":"127.0.0.1:8301","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899183} /tidb/cdc/default/__cdc_meta__/capture/890e253d-8a87-438d-afc6-63906c93969f {"id":"890e253d-8a87-438d-afc6-63906c93969f","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899178} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/1f5d96087d955add 890e253d-8a87-438d-afc6-63906c93969f /tidb/cdc/default/__cdc_meta__/owner/1f5d96087d955b1d 1959bee5-3b2f-41f2-90a9-3dc06476a405 /tidb/cdc/default/default/upstream/7489989914123752837 {"id":7489989914123752837,"pd-endpoints":"https://127.0.0.1:2579,https://127.0.0.1:2579","key-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/server-key.pem","cert-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/server.pem","ca-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem","cert-allowed-cn":["client","tidb-server"]}' + grep -q 'etcd info' + break + set +x [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } + start_proxy + echo 'dumpling grpc packet to /tmp/tidb_cdc_test/http_proxies/test_proxy.log...' dumpling grpc packet to /tmp/tidb_cdc_test/http_proxies/test_proxy.log... + proxy_pid=9345 + echo 'proxy port: 13444' proxy port: 13444 + echo 'proxy pid: 9345' proxy pid: 9345 + sleep 5 + GO111MODULE=on + WORK_DIR=/tmp/tidb_cdc_test/http_proxies + go run /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/http_proxies/run-proxy.go --port=13444 go: downloading go1.23.4 (linux/amd64) Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2025/04/06 08:26:27.983 +08:00] [INFO] [main.go:86] ["running ddl test: 2 addDropColumnDDL"] [2025/04/06 08:26:28.245 +08:00] [INFO] [main.go:221] ["0 insert success: 100"] [2025/04/06 08:26:28.246 +08:00] [INFO] [main.go:221] ["1 insert success: 100"] [2025/04/06 08:26:28.478 +08:00] [INFO] [main.go:235] ["0 delete success: 100"] [2025/04/06 08:26:28.479 +08:00] [INFO] [main.go:221] ["0 insert success: 200"] [2025/04/06 08:26:28.479 +08:00] [INFO] [main.go:221] ["1 insert success: 200"] [2025/04/06 08:26:28.482 +08:00] [INFO] [main.go:235] ["1 delete success: 100"] [2025/04/06 08:26:28.721 +08:00] [INFO] [main.go:221] ["1 insert success: 300"] [2025/04/06 08:26:28.726 +08:00] [INFO] [main.go:221] ["0 insert success: 300"] pass test: check health pass test: get status [2025/04/06 08:26:29.066 +08:00] [INFO] [main.go:221] ["1 insert success: 400"] [2025/04/06 08:26:29.069 +08:00] [INFO] [main.go:235] ["1 delete success: 200"] [2025/04/06 08:26:29.069 +08:00] [INFO] [main.go:235] ["0 delete success: 200"] [2025/04/06 08:26:29.071 +08:00] [INFO] [main.go:221] ["0 insert success: 400"] [2025/04/06 08:26:29.299 +08:00] [INFO] [main.go:221] ["1 insert success: 500"] [2025/04/06 08:26:29.303 +08:00] [INFO] [main.go:221] ["0 insert success: 500"] [2025/04/06 08:26:29.530 +08:00] [INFO] [main.go:221] ["1 insert success: 600"] [2025/04/06 08:26:29.532 +08:00] [INFO] [main.go:235] ["0 delete success: 300"] [2025/04/06 08:26:29.532 +08:00] [INFO] [main.go:235] ["1 delete success: 300"] [2025/04/06 08:26:29.533 +08:00] [INFO] [main.go:221] ["0 insert success: 600"] pass test: create changefeed check_changefeed_state https://127.0.0.1:2579 changefeed-test1 normal null /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates + endpoints=https://127.0.0.1:2579 + changefeed_id=changefeed-test1 + expected_state=normal + error_msg=null + tls_dir=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates + [[ https://127.0.0.1:2579 =~ https ]] ++ cdc cli changefeed query --ca=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client.pem --key=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client-key.pem --pd=https://127.0.0.1:2579 -c changefeed-test1 -s [2025/04/06 08:26:29.765 +08:00] [INFO] [main.go:221] ["0 insert success: 700"] [2025/04/06 08:26:29.767 +08:00] [INFO] [main.go:221] ["1 insert success: 700"] + info='{ "upstream_id": 7489989914123752837, "namespace": "default", "id": "changefeed-test1", "state": "normal", "checkpoint_tso": 457152709001478157, "checkpoint_time": "2025-04-06 08:26:29.001", "error": null }' + echo '{ "upstream_id": 7489989914123752837, "namespace": "default", "id": "changefeed-test1", "state": "normal", "checkpoint_tso": 457152709001478157, "checkpoint_time": "2025-04-06 08:26:29.001", "error": null }' { "upstream_id": 7489989914123752837, "namespace": "default", "id": "changefeed-test1", "state": "normal", "checkpoint_tso": 457152709001478157, "checkpoint_time": "2025-04-06 08:26:29.001", "error": null } ++ echo '{' '"upstream_id":' 7489989914123752837, '"namespace":' '"default",' '"id":' '"changefeed-test1",' '"state":' '"normal",' '"checkpoint_tso":' 457152709001478157, '"checkpoint_time":' '"2025-04-06' '08:26:29.001",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7489989914123752837, '"namespace":' '"default",' '"id":' '"changefeed-test1",' '"state":' '"normal",' '"checkpoint_tso":' 457152709001478157, '"checkpoint_time":' '"2025-04-06' '08:26:29.001",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully check_changefeed_state https://127.0.0.1:2579 changefeed-test2 normal null /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates + endpoints=https://127.0.0.1:2579 + changefeed_id=changefeed-test2 + expected_state=normal + error_msg=null + tls_dir=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates + [[ https://127.0.0.1:2579 =~ https ]] ++ cdc cli changefeed query --ca=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client.pem --key=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client-key.pem --pd=https://127.0.0.1:2579 -c changefeed-test2 -s [2025/04/06 08:26:30.000 +08:00] [INFO] [main.go:221] ["1 insert success: 800"] [2025/04/06 08:26:30.002 +08:00] [INFO] [main.go:235] ["1 delete success: 400"] [2025/04/06 08:26:30.007 +08:00] [INFO] [main.go:235] ["0 delete success: 400"] [2025/04/06 08:26:30.008 +08:00] [INFO] [main.go:221] ["0 insert success: 800"] + info='{ "upstream_id": 7489989914123752837, "namespace": "default", "id": "changefeed-test2", "state": "normal", "checkpoint_tso": 457152709040799761, "checkpoint_time": "2025-04-06 08:26:29.151", "error": null }' + echo '{ "upstream_id": 7489989914123752837, "namespace": "default", "id": "changefeed-test2", "state": "normal", "checkpoint_tso": 457152709040799761, "checkpoint_time": "2025-04-06 08:26:29.151", "error": null }' { "upstream_id": 7489989914123752837, "namespace": "default", "id": "changefeed-test2", "state": "normal", "checkpoint_tso": 457152709040799761, "checkpoint_time": "2025-04-06 08:26:29.151", "error": null } ++ echo '{' '"upstream_id":' 7489989914123752837, '"namespace":' '"default",' '"id":' '"changefeed-test2",' '"state":' '"normal",' '"checkpoint_tso":' 457152709040799761, '"checkpoint_time":' '"2025-04-06' '08:26:29.151",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7489989914123752837, '"namespace":' '"default",' '"id":' '"changefeed-test2",' '"state":' '"normal",' '"checkpoint_tso":' 457152709040799761, '"checkpoint_time":' '"2025-04-06' '08:26:29.151",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully check_changefeed_state https://127.0.0.1:2579 changefeed-test3 normal null /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates + endpoints=https://127.0.0.1:2579 + changefeed_id=changefeed-test3 + expected_state=normal + error_msg=null + tls_dir=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates + [[ https://127.0.0.1:2579 =~ https ]] ++ cdc cli changefeed query --ca=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client.pem --key=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client-key.pem --pd=https://127.0.0.1:2579 -c changefeed-test3 -s [2025/04/06 08:26:30.229 +08:00] [INFO] [main.go:221] ["1 insert success: 900"] [2025/04/06 08:26:30.236 +08:00] [INFO] [main.go:221] ["0 insert success: 900"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + info='{ "upstream_id": 7489989914123752837, "namespace": "default", "id": "changefeed-test3", "state": "normal", "checkpoint_tso": 457152709066752019, "checkpoint_time": "2025-04-06 08:26:29.250", "error": null }' + echo '{ "upstream_id": 7489989914123752837, "namespace": "default", "id": "changefeed-test3", "state": "normal", "checkpoint_tso": 457152709066752019, "checkpoint_time": "2025-04-06 08:26:29.250", "error": null }' { "upstream_id": 7489989914123752837, "namespace": "default", "id": "changefeed-test3", "state": "normal", "checkpoint_tso": 457152709066752019, "checkpoint_time": "2025-04-06 08:26:29.250", "error": null } ++ echo '{' '"upstream_id":' 7489989914123752837, '"namespace":' '"default",' '"id":' '"changefeed-test3",' '"state":' '"normal",' '"checkpoint_tso":' 457152709066752019, '"checkpoint_time":' '"2025-04-06' '08:26:29.250",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7489989914123752837, '"namespace":' '"default",' '"id":' '"changefeed-test3",' '"state":' '"normal",' '"checkpoint_tso":' 457152709066752019, '"checkpoint_time":' '"2025-04-06' '08:26:29.250",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully check_changefeed_state https://127.0.0.1:2579 changefeed-test4 normal null /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates + endpoints=https://127.0.0.1:2579 + changefeed_id=changefeed-test4 + expected_state=normal + error_msg=null + tls_dir=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates + [[ https://127.0.0.1:2579 =~ https ]] ++ cdc cli changefeed query --ca=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client.pem --key=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client-key.pem --pd=https://127.0.0.1:2579 -c changefeed-test4 -s [2025/04/06 08:26:30.448 +08:00] [INFO] [main.go:221] ["1 insert success: 1000"] [2025/04/06 08:26:30.451 +08:00] [INFO] [main.go:235] ["1 delete success: 500"] [2025/04/06 08:26:30.455 +08:00] [INFO] [main.go:235] ["0 delete success: 500"] [2025/04/06 08:26:30.456 +08:00] [INFO] [main.go:221] ["0 insert success: 1000"] + info='{ "upstream_id": 7489989914123752837, "namespace": "default", "id": "changefeed-test4", "state": "normal", "checkpoint_tso": 457152709092966417, "checkpoint_time": "2025-04-06 08:26:29.350", "error": null }' + echo '{ "upstream_id": 7489989914123752837, "namespace": "default", "id": "changefeed-test4", "state": "normal", "checkpoint_tso": 457152709092966417, "checkpoint_time": "2025-04-06 08:26:29.350", "error": null }' { "upstream_id": 7489989914123752837, "namespace": "default", "id": "changefeed-test4", "state": "normal", "checkpoint_tso": 457152709092966417, "checkpoint_time": "2025-04-06 08:26:29.350", "error": null } ++ echo '{' '"upstream_id":' 7489989914123752837, '"namespace":' '"default",' '"id":' '"changefeed-test4",' '"state":' '"normal",' '"checkpoint_tso":' 457152709092966417, '"checkpoint_time":' '"2025-04-06' '08:26:29.350",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7489989914123752837, '"namespace":' '"default",' '"id":' '"changefeed-test4",' '"state":' '"normal",' '"checkpoint_tso":' 457152709092966417, '"checkpoint_time":' '"2025-04-06' '08:26:29.350",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully [2025/04/06 08:26:30.672 +08:00] [INFO] [main.go:221] ["1 insert success: 1100"] [2025/04/06 08:26:30.681 +08:00] [INFO] [main.go:221] ["0 insert success: 1100"] =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/synced_status/run.sh using Sink-Type: mysql... <<================= +++ dirname /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/synced_status/run.sh ++ cd /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/synced_status ++ pwd + CUR=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/synced_status + source /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/synced_status/../_utils/test_prepare ++ UP_TIDB_HOST=127.0.0.1 ++ UP_TIDB_PORT=4000 ++ UP_TIDB_OTHER_PORT=4001 ++ UP_TIDB_STATUS=10080 ++ UP_TIDB_OTHER_STATUS=10081 ++ DOWN_TIDB_HOST=127.0.0.1 ++ DOWN_TIDB_PORT=3306 ++ DOWN_TIDB_STATUS=20080 ++ TLS_TIDB_HOST=127.0.0.1 ++ TLS_TIDB_PORT=3307 ++ TLS_TIDB_STATUS=30080 ++ UP_PD_HOST_1=127.0.0.1 ++ UP_PD_PORT_1=2379 ++ UP_PD_PEER_PORT_1=2380 ++ UP_PD_HOST_2=127.0.0.1 ++ UP_PD_PORT_2=2679 ++ UP_PD_PEER_PORT_2=2680 ++ UP_PD_HOST_3=127.0.0.1 ++ UP_PD_PORT_3=2779 ++ UP_PD_PEER_PORT_3=2780 ++ DOWN_PD_HOST=127.0.0.1 ++ DOWN_PD_PORT=2479 ++ DOWN_PD_PEER_PORT=2480 ++ TLS_PD_HOST=127.0.0.1 ++ TLS_PD_PORT=2579 ++ TLS_PD_PEER_PORT=2580 ++ UP_TIKV_HOST_1=127.0.0.1 ++ UP_TIKV_PORT_1=20160 ++ UP_TIKV_STATUS_PORT_1=20181 ++ UP_TIKV_HOST_2=127.0.0.1 ++ UP_TIKV_PORT_2=20161 ++ UP_TIKV_STATUS_PORT_2=20182 ++ UP_TIKV_HOST_3=127.0.0.1 ++ UP_TIKV_PORT_3=20162 ++ UP_TIKV_STATUS_PORT_3=20183 ++ DOWN_TIKV_HOST=127.0.0.1 ++ DOWN_TIKV_PORT=21160 ++ DOWN_TIKV_STATUS_PORT=21180 ++ TLS_TIKV_HOST=127.0.0.1 ++ TLS_TIKV_PORT=22160 ++ TLS_TIKV_STATUS_PORT=22180 +++ cat /tmp/tidb_cdc_test/KAFKA_VERSION +++ echo 2.4.1 ++ KAFKA_VERSION=2.4.1 + WORK_DIR=/tmp/tidb_cdc_test/synced_status + CDC_BINARY=cdc.test + SINK_TYPE=mysql + CDC_COUNT=3 + DB_COUNT=4 + trap stop_tidb_cluster EXIT + run_normal_case_and_unavailable_pd conf/changefeed.toml + rm -rf /tmp/tidb_cdc_test/synced_status + mkdir -p /tmp/tidb_cdc_test/synced_status + start_tidb_cluster --workdir /tmp/tidb_cdc_test/synced_status The 1 times to try to start tidb cluster... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/autorandom/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [2025/04/06 08:26:30.932 +08:00] [INFO] [main.go:221] ["1 insert success: 1200"] [2025/04/06 08:26:30.934 +08:00] [INFO] [main.go:235] ["1 delete success: 600"] [2025/04/06 08:26:30.947 +08:00] [INFO] [main.go:235] ["0 delete success: 600"] [2025/04/06 08:26:30.948 +08:00] [INFO] [main.go:221] ["0 insert success: 1200"] [2025/04/06 08:26:31.274 +08:00] [INFO] [main.go:221] ["1 insert success: 1300"] [2025/04/06 08:26:31.294 +08:00] [INFO] [main.go:221] ["0 insert success: 1300"] [2025/04/06 08:26:31.484 +08:00] [INFO] [main.go:221] ["1 insert success: 1400"] [2025/04/06 08:26:31.486 +08:00] [INFO] [main.go:235] ["1 delete success: 700"] [2025/04/06 08:26:31.507 +08:00] [INFO] [main.go:235] ["0 delete success: 700"] [2025/04/06 08:26:31.508 +08:00] [INFO] [main.go:221] ["0 insert success: 1400"] [2025/04/06 08:26:31.694 +08:00] [INFO] [main.go:221] ["1 insert success: 1500"] [2025/04/06 08:26:31.728 +08:00] [INFO] [main.go:221] ["0 insert success: 1500"] [2025/04/06 08:26:31.911 +08:00] [INFO] [main.go:221] ["1 insert success: 1600"] [2025/04/06 08:26:31.914 +08:00] [INFO] [main.go:235] ["1 delete success: 800"] [2025/04/06 08:26:31.945 +08:00] [INFO] [main.go:235] ["0 delete success: 800"] [2025/04/06 08:26:31.946 +08:00] [INFO] [main.go:221] ["0 insert success: 1600"] [2025/04/06 08:26:32.121 +08:00] [INFO] [main.go:221] ["1 insert success: 1700"] [2025/04/06 08:26:32.163 +08:00] [INFO] [main.go:221] ["0 insert success: 1700"] [2025/04/06 08:26:32.325 +08:00] [INFO] [main.go:221] ["1 insert success: 1800"] [2025/04/06 08:26:32.327 +08:00] [INFO] [main.go:235] ["1 delete success: 900"] go: downloading google.golang.org/grpc v1.64.0 go: downloading github.com/pingcap/log v1.1.1-0.20241212030209-7e3ff8601a2a go: downloading go.uber.org/zap v1.27.0 go: downloading github.com/bradleyjkemp/grpc-tools v0.2.5 go: downloading gopkg.in/natefinch/lumberjack.v2 v2.2.1 go: downloading github.com/improbable-eng/grpc-web v0.12.0 go: downloading golang.org/x/net v0.33.0 go: downloading github.com/pkg/errors v0.9.1 go: downloading github.com/sirupsen/logrus v1.9.3 go: downloading golang.org/x/sys v0.28.0 go: downloading go.uber.org/multierr v1.11.0 + export http_proxy=http://127.0.0.1:13444 + http_proxy=http://127.0.0.1:13444 + export https_proxy=http://127.0.0.1:13444 + https_proxy=http://127.0.0.1:13444 + echo 'try to connect pd cluster via proxy, pd addr: 10.233.76.109:2379' try to connect pd cluster via proxy, pd addr: 10.233.76.109:2379 + ensure 10 curl http://10.233.76.109:2379/ curl http://10.233.76.109:2379/ % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0curl: (7) Failed connect to 127.0.0.1:13444; Connection refused run task failed 1-th time, retry later go: downloading github.com/rs/cors v1.7.0 go: downloading github.com/gorilla/websocket v1.5.1 go: downloading github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f go: downloading golang.org/x/text v0.21.0 go: downloading google.golang.org/genproto/googleapis/rpc v0.0.0-20240515191416-fc5f0ca64291 go: downloading google.golang.org/protobuf v1.35.2 [2025/04/06 08:26:32.381 +08:00] [INFO] [main.go:235] ["0 delete success: 900"] [2025/04/06 08:26:32.382 +08:00] [INFO] [main.go:221] ["0 insert success: 1800"] [2025/04/06 08:26:32.524 +08:00] [INFO] [main.go:221] ["1 insert success: 1900"] [2025/04/06 08:26:32.589 +08:00] [INFO] [main.go:221] ["0 insert success: 1900"] go: downloading google.golang.org/genproto v0.0.0-20240401170217-c3f982113cda [2025/04/06 08:26:32.733 +08:00] [INFO] [main.go:221] ["1 insert success: 2000"] [2025/04/06 08:26:32.736 +08:00] [INFO] [main.go:235] ["1 delete success: 1000"] [2025/04/06 08:26:32.804 +08:00] [INFO] [main.go:235] ["0 delete success: 1000"] [2025/04/06 08:26:32.805 +08:00] [INFO] [main.go:221] ["0 insert success: 2000"] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489989963316572036 TiDB Cluster ID. tikv_gc_leader_uuid 65821f792c0000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-9-92vm1, pid:15485, start at 2025-04-06 08:26:30.48241608 +0800 CST m=+2.201823817 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:28:30.536 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:26:30.499 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:16:30.499 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489989963316572036 TiDB Cluster ID. tikv_gc_leader_uuid 65821f792c0000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-9-92vm1, pid:15485, start at 2025-04-06 08:26:30.48241608 +0800 CST m=+2.201823817 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:28:30.536 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:26:30.499 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:16:30.499 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489989967417189986 TiDB Cluster ID. tikv_gc_leader_uuid 65821f79440000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-9-92vm1, pid:15563, start at 2025-04-06 08:26:30.570228829 +0800 CST m=+2.223313305 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:28:30.577 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:26:30.544 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:16:30.544 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/server_config_compatibility/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/server_config_compatibility/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/server_config_compatibility/tiflash/log/proxy.log"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/server_config_compatibility/tiflash-proxy.toml"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/server_config_compatibility/tiflash/db/proxy"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } start tidb cluster in /tmp/tidb_cdc_test/autorandom Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... table test.simple1 exists running list_changefeed start tidb cluster in /tmp/tidb_cdc_test/synced_status Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... pass test: list changefeed curl http://10.233.76.109:2379/ % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0curl: (7) Failed connect to 127.0.0.1:13444; Connection refused run task failed 2-th time, retry later Run finished, takes 8.74590759s INSERT_ERROR - Takes(s): 7.7, Count: 100, OPS: 13.0, Avg(us): 271681, Min(us): 113218, Max(us): 592505, 95th(us): 504000, 99th(us): 579000 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.server_config_compatibility.cli.16974.out cli tso query --pd=http://127.0.0.1:2379 running get_changefeed table consistent_replicate_storage_file_large_value.usertable exists table consistent_replicate_storage_file_large_value.check1 not exists for 1-th check, retry later pass test: get changefeed + set +x + tso='457152710560448513 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 457152710560448513 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Sun Apr 6 08:26:36 CST 2025] <<<<<< START cdc server in server_config_compatibility case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.server_config_compatibility.1701117013.out server --log-file /tmp/tidb_cdc_test/server_config_compatibility/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/server_config_compatibility/cdc_data --cluster-id default --config /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/server_config_compatibility/conf/server.toml ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 running pause_changefeed Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table consistent_replicate_storage_file_large_value.check1 exists table consistent_replicate_storage_file_large_value.t2 not exists for 1-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release pass test: pause changefeed curl http://10.233.76.109:2379/ % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0curl: (7) Failed connect to 127.0.0.1:13444; Connection refused run task failed 3-th time, retry later Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) running update_changefeed [2025/04/06 08:26:38.891 +08:00] [INFO] [main.go:86] ["running ddl test: 3 addDropColumnDDL2"] [2025/04/06 08:26:39.116 +08:00] [INFO] [main.go:221] ["1 insert success: 100"] [2025/04/06 08:26:39.127 +08:00] [INFO] [main.go:221] ["0 insert success: 100"] [2025/04/06 08:26:39.304 +08:00] [INFO] [main.go:221] ["1 insert success: 200"] [2025/04/06 08:26:39.306 +08:00] [INFO] [main.go:235] ["1 delete success: 100"] [2025/04/06 08:26:39.318 +08:00] [INFO] [main.go:235] ["0 delete success: 100"] [2025/04/06 08:26:39.319 +08:00] [INFO] [main.go:221] ["0 insert success: 200"] pass test: update changefeed [2025/04/06 08:26:39.504 +08:00] [INFO] [main.go:221] ["1 insert success: 300"] [2025/04/06 08:26:39.519 +08:00] [INFO] [main.go:221] ["0 insert success: 300"] [2025/04/06 08:26:39.696 +08:00] [INFO] [main.go:221] ["1 insert success: 400"] [2025/04/06 08:26:39.698 +08:00] [INFO] [main.go:235] ["1 delete success: 200"] [2025/04/06 08:26:39.716 +08:00] [INFO] [main.go:235] ["0 delete success: 200"] [2025/04/06 08:26:39.717 +08:00] [INFO] [main.go:221] ["0 insert success: 400"] [2025/04/06 08:26:39.894 +08:00] [INFO] [main.go:221] ["1 insert success: 500"] [2025/04/06 08:26:39.913 +08:00] [INFO] [main.go:221] ["0 insert success: 500"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 06 Apr 2025 00:26:39 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/39c55018-f0dd-4c25-9260-ea4b0a2767a7 {"id":"39c55018-f0dd-4c25-9260-ea4b0a2767a7","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899196} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087dc630f7 39c55018-f0dd-4c25-9260-ea4b0a2767a7 /tidb/cdc/default/default/upstream/7489989963316572036 {"id":7489989963316572036,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/39c55018-f0dd-4c25-9260-ea4b0a2767a7 {"id":"39c55018-f0dd-4c25-9260-ea4b0a2767a7","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899196} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087dc630f7 39c55018-f0dd-4c25-9260-ea4b0a2767a7 /tidb/cdc/default/default/upstream/7489989963316572036 {"id":7489989963316572036,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/39c55018-f0dd-4c25-9260-ea4b0a2767a7 {"id":"39c55018-f0dd-4c25-9260-ea4b0a2767a7","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899196} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087dc630f7 39c55018-f0dd-4c25-9260-ea4b0a2767a7 /tidb/cdc/default/default/upstream/7489989963316572036 {"id":7489989963316572036,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.server_config_compatibility.cli.17071.out cli changefeed create --start-ts=457152710560448513 --sink-uri=mysql+ssl://normal:123456@127.0.0.1:3306/ table consistent_replicate_storage_file_large_value.t2 not exists for 2-th check, retry later Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... [2025/04/06 08:26:40.091 +08:00] [INFO] [main.go:221] ["1 insert success: 600"] [2025/04/06 08:26:40.093 +08:00] [INFO] [main.go:235] ["1 delete success: 300"] [2025/04/06 08:26:40.110 +08:00] [INFO] [main.go:235] ["0 delete success: 300"] [2025/04/06 08:26:40.111 +08:00] [INFO] [main.go:221] ["0 insert success: 600"] Create changefeed successfully! ID: 127ff1f5-e2b0-4bb3-9dc1-d02c223a1a4e Info: {"upstream_id":7489989963316572036,"namespace":"default","id":"127ff1f5-e2b0-4bb3-9dc1-d02c223a1a4e","sink_uri":"mysql+ssl://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-06T08:26:40.003735771+08:00","start_ts":457152710560448513,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457152710560448513,"checkpoint_ts":457152710560448513,"checkpoint_time":"2025-04-06 08:26:34.948"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2025/04/06 08:26:40.279 +08:00] [INFO] [main.go:221] ["1 insert success: 700"] [2025/04/06 08:26:40.302 +08:00] [INFO] [main.go:221] ["0 insert success: 700"] [2025/04/06 08:26:40.464 +08:00] [INFO] [main.go:221] ["1 insert success: 800"] [2025/04/06 08:26:40.467 +08:00] [INFO] [main.go:235] ["1 delete success: 400"] [2025/04/06 08:26:40.493 +08:00] [INFO] [main.go:235] ["0 delete success: 400"] [2025/04/06 08:26:40.494 +08:00] [INFO] [main.go:221] ["0 insert success: 800"] [2025/04/06 08:26:40.665 +08:00] [INFO] [main.go:221] ["1 insert success: 900"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ci_collation_compatibility.t exists table ci_collation_compatibility.finish_mark_1 exists check diff successfully [2025/04/06 08:26:40.692 +08:00] [INFO] [main.go:221] ["0 insert success: 900"] [2025/04/06 08:26:40.862 +08:00] [INFO] [main.go:221] ["1 insert success: 1000"] [2025/04/06 08:26:40.864 +08:00] [INFO] [main.go:235] ["1 delete success: 500"] [2025/04/06 08:26:40.925 +08:00] [INFO] [main.go:235] ["0 delete success: 500"] [2025/04/06 08:26:40.927 +08:00] [INFO] [main.go:221] ["0 insert success: 1000"] running resume_changefeed wait process cdc.test exit for 1-th time... [2025/04/06 08:26:41.229 +08:00] [INFO] [main.go:221] ["1 insert success: 1100"] [2025/04/06 08:26:41.248 +08:00] [INFO] [main.go:221] ["0 insert success: 1100"] [2025/04/06 08:26:41.433 +08:00] [INFO] [main.go:221] ["1 insert success: 1200"] [2025/04/06 08:26:41.435 +08:00] [INFO] [main.go:235] ["1 delete success: 600"] [2025/04/06 08:26:41.462 +08:00] [INFO] [main.go:235] ["0 delete success: 600"] [2025/04/06 08:26:41.463 +08:00] [INFO] [main.go:221] ["0 insert success: 1200"] + set +x wait process cdc.test exit for 2-th time... TEST FAILED: OUTPUT DOES NOT CONTAIN 'id: 1' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ check data failed 1-th time, retry later table consistent_replicate_storage_file_large_value.t2 not exists for 3-th check, retry later [2025/04/06 08:26:41.750 +08:00] [INFO] [main.go:221] ["1 insert success: 1300"] [2025/04/06 08:26:41.774 +08:00] [INFO] [main.go:221] ["0 insert success: 1300"] [2025/04/06 08:26:41.950 +08:00] [INFO] [main.go:221] ["1 insert success: 1400"] [2025/04/06 08:26:41.952 +08:00] [INFO] [main.go:235] ["1 delete success: 700"] [2025/04/06 08:26:41.972 +08:00] [INFO] [main.go:235] ["0 delete success: 700"] [2025/04/06 08:26:41.973 +08:00] [INFO] [main.go:221] ["0 insert success: 1400"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Sun Apr 6 08:26:42 CST 2025] <<<<<< run test case ci_collation_compatibility success! >>>>>> [2025/04/06 08:26:42.240 +08:00] [INFO] [main.go:221] ["1 insert success: 1500"] [2025/04/06 08:26:42.266 +08:00] [INFO] [main.go:221] ["0 insert success: 1500"] pass test: resume changefeed [2025/04/06 08:26:42.474 +08:00] [INFO] [main.go:221] ["1 insert success: 1600"] [2025/04/06 08:26:42.477 +08:00] [INFO] [main.go:235] ["1 delete success: 800"] [2025/04/06 08:26:42.497 +08:00] [INFO] [main.go:235] ["0 delete success: 800"] [2025/04/06 08:26:42.498 +08:00] [INFO] [main.go:221] ["0 insert success: 1600"] [2025/04/06 08:26:42.840 +08:00] [INFO] [main.go:221] ["1 insert success: 1700"] [2025/04/06 08:26:42.873 +08:00] [INFO] [main.go:221] ["0 insert success: 1700"] [2025/04/06 08:26:43.049 +08:00] [INFO] [main.go:221] ["1 insert success: 1800"] [2025/04/06 08:26:43.051 +08:00] [INFO] [main.go:235] ["1 delete success: 900"] [2025/04/06 08:26:43.083 +08:00] [INFO] [main.go:235] ["0 delete success: 900"] [2025/04/06 08:26:43.083 +08:00] [INFO] [main.go:221] ["0 insert success: 1800"] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489990012939363297 TiDB Cluster ID. tikv_gc_leader_uuid 65821f838400005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-j-vgmw9, pid:28243, start at 2025-04-06 08:26:41.047606305 +0800 CST m=+2.361073142 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:28:41.056 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:26:41.040 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:16:41.040 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489990012939363297 TiDB Cluster ID. tikv_gc_leader_uuid 65821f838400005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-j-vgmw9, pid:28243, start at 2025-04-06 08:26:41.047606305 +0800 CST m=+2.361073142 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:28:41.056 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:26:41.040 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:16:41.040 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489990009231948571 TiDB Cluster ID. tikv_gc_leader_uuid 65821f83838000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-j-vgmw9, pid:28320, start at 2025-04-06 08:26:41.081594846 +0800 CST m=+2.324369823 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:28:41.090 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:26:41.088 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:16:41.088 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/autorandom/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/autorandom/tiflash/log/error.log arg matches is ArgMatches { args: {"pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/autorandom/tiflash-proxy.toml"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/autorandom/tiflash/log/proxy.log"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/autorandom/tiflash/db/proxy"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [2025/04/06 08:26:43.363 +08:00] [INFO] [main.go:221] ["1 insert success: 1900"] [2025/04/06 08:26:43.446 +08:00] [INFO] [main.go:221] ["0 insert success: 1900"] [2025/04/06 08:26:43.616 +08:00] [INFO] [main.go:221] ["1 insert success: 2000"] [2025/04/06 08:26:43.618 +08:00] [INFO] [main.go:235] ["1 delete success: 1000"] running rebalance_table pass test: rebalance table TEST FAILED: OUTPUT DOES NOT CONTAIN 'id: 1' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ check data failed 2-th time, retry later table consistent_replicate_storage_file_large_value.t2 not exists for 4-th check, retry later [2025/04/06 08:26:43.651 +08:00] [INFO] [main.go:235] ["0 delete success: 1000"] [2025/04/06 08:26:43.652 +08:00] [INFO] [main.go:221] ["0 insert success: 2000"] [2025/04/06 08:26:43.817 +08:00] [INFO] [main.go:221] ["1 insert success: 2100"] [2025/04/06 08:26:43.855 +08:00] [INFO] [main.go:221] ["0 insert success: 2100"] running list_processor VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489990016759345567 TiDB Cluster ID. tikv_gc_leader_uuid 65821f8512c0033 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-c-g3td9, pid:16124, start at 2025-04-06 08:26:42.649884066 +0800 CST m=+2.546428000 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:28:42.657 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:26:42.635 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:16:42.635 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489990016759345567 TiDB Cluster ID. tikv_gc_leader_uuid 65821f8512c0033 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-c-g3td9, pid:16124, start at 2025-04-06 08:26:42.649884066 +0800 CST m=+2.546428000 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:28:42.657 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:26:42.635 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:16:42.635 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489990013742826578 TiDB Cluster ID. tikv_gc_leader_uuid 65821f852a80006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-c-g3td9, pid:16187, start at 2025-04-06 08:26:42.738828179 +0800 CST m=+2.561569277 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:28:42.746 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:26:42.730 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:16:42.730 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/synced_status/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/synced_status/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash/db/proxy"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } pass test: list processors [Sun Apr 6 08:26:45 CST 2025] <<<<<< START cdc server in autorandom case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.autorandom.2971029712.out server --log-file /tmp/tidb_cdc_test/autorandom/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/autorandom/cdc_data --cluster-id default + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 curl http://10.233.76.109:2379/ % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0curl: (7) Failed connect to 127.0.0.1:13444; Connection refused run task failed 4-th time, retry later table consistent_replicate_storage_file_large_value.t2 not exists for 5-th check, retry later running get_processor + '[' mysql == mysql ']' + check_table_exists test.t2 127.0.0.1 3306 table test.t2 not exists for 1-th check, retry later table test.t2 exists + sleep 10 check data successfully time="2025-04-06T08:26:46+08:00" level=info msg="Listening on 127.0.0.1:13444" time="2025-04-06T08:26:46+08:00" level=info msg="Not intercepting TLS connections" wait process cdc.test exit for 1-th time... + cd /tmp/tidb_cdc_test/synced_status ++ run_cdc_cli_tso_query 127.0.0.1 2379 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.17432.out cli tso query --pd=http://127.0.0.1:2379 wait process cdc.test exit for 2-th time... table consistent_replicate_storage_file_large_value.t2 not exists for 6-th check, retry later cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Sun Apr 6 08:26:47 CST 2025] <<<<<< run test case server_config_compatibility success! >>>>>> pass test: get processor + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 06 Apr 2025 00:26:48 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c4e256d7-b498-41dd-a3ca-878068b4f0c9 {"id":"c4e256d7-b498-41dd-a3ca-878068b4f0c9","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899205} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087df0edd8 c4e256d7-b498-41dd-a3ca-878068b4f0c9 /tidb/cdc/default/default/upstream/7489990012939363297 {"id":7489990012939363297,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c4e256d7-b498-41dd-a3ca-878068b4f0c9 {"id":"c4e256d7-b498-41dd-a3ca-878068b4f0c9","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899205} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087df0edd8 c4e256d7-b498-41dd-a3ca-878068b4f0c9 /tidb/cdc/default/default/upstream/7489990012939363297 {"id":7489990012939363297,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c4e256d7-b498-41dd-a3ca-878068b4f0c9 {"id":"c4e256d7-b498-41dd-a3ca-878068b4f0c9","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899205} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087df0edd8 c4e256d7-b498-41dd-a3ca-878068b4f0c9 /tidb/cdc/default/default/upstream/7489990012939363297 {"id":7489990012939363297,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: 3d9a5ecd-96e5-43ea-b5c9-5ff2fd0cb539 Info: {"upstream_id":7489990012939363297,"namespace":"default","id":"3d9a5ecd-96e5-43ea-b5c9-5ff2fd0cb539","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-06T08:26:48.281985114+08:00","start_ts":457152714031497221,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457152714031497221,"checkpoint_ts":457152714031497221,"checkpoint_time":"2025-04-06 08:26:48.189"} table autorandom_test.table_a not exists for 1-th check, retry later + set +x + tso='457152713676554241 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 457152713676554241 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + start_ts=457152713676554241 + run_cdc_server --workdir /tmp/tidb_cdc_test/synced_status --binary cdc.test [Sun Apr 6 08:26:48 CST 2025] <<<<<< START cdc server in synced_status case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.1747017472.out server --log-file /tmp/tidb_cdc_test/synced_status/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/synced_status/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 running move_table pass test: move table table consistent_replicate_storage_file_large_value.t2 exists check diff successfully [2025/04/06 08:26:49.772 +08:00] [INFO] [main.go:86] ["running ddl test: 4 modifyColumnDDL"] [2025/04/06 08:26:50.193 +08:00] [INFO] [main.go:221] ["0 insert success: 100"] [2025/04/06 08:26:50.207 +08:00] [INFO] [main.go:221] ["1 insert success: 100"] wait process cdc.test exit for 1-th time... [2025/04/06 08:26:50.447 +08:00] [INFO] [main.go:221] ["0 insert success: 200"] [2025/04/06 08:26:50.450 +08:00] [INFO] [main.go:235] ["0 delete success: 100"] [2025/04/06 08:26:50.459 +08:00] [INFO] [main.go:235] ["1 delete success: 100"] [2025/04/06 08:26:50.460 +08:00] [INFO] [main.go:221] ["1 insert success: 200"] running set_log_level pass test: set log level [2025/04/06 08:26:50.861 +08:00] [INFO] [main.go:221] ["0 insert success: 300"] [2025/04/06 08:26:50.875 +08:00] [INFO] [main.go:221] ["1 insert success: 300"] table autorandom_test.table_a exists check diff failed 1-th time, retry later [2025/04/06 08:26:51.161 +08:00] [INFO] [main.go:221] ["0 insert success: 400"] [2025/04/06 08:26:51.163 +08:00] [INFO] [main.go:235] ["0 delete success: 200"] wait process cdc.test exit for 2-th time... + run_sql 'SELECT * from test.t2' 127.0.0.1 3306 + check_contains 'a: 2023-02-08' + stop_tidb_cluster [2025/04/06 08:26:51.252 +08:00] [INFO] [main.go:235] ["1 delete success: 200"] [2025/04/06 08:26:51.254 +08:00] [INFO] [main.go:221] ["1 insert success: 400"] wait process cdc.test exit for 3-th time... process cdc.test already exit [Sun Apr 6 08:26:51 CST 2025] <<<<<< START cdc server in consistent_replicate_storage_file_large_value case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sink/dmlsink/txn/mysql/MySQLSinkHangLongTime=return(true)' + '[' -z '' ']' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.consistent_replicate_storage_file_large_value.1454714549.out server --log-file /tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/cdc_data --cluster-id default + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2025/04/06 08:26:51.501 +08:00] [INFO] [main.go:221] ["0 insert success: 500"] [2025/04/06 08:26:51.533 +08:00] [INFO] [main.go:221] ["1 insert success: 500"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 06 Apr 2025 00:26:51 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6a508f52-fbeb-437d-89a4-cdae39086298 {"id":"6a508f52-fbeb-437d-89a4-cdae39086298","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899208} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087df278e8 6a508f52-fbeb-437d-89a4-cdae39086298 /tidb/cdc/default/default/upstream/7489990016759345567 {"id":7489990016759345567,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6a508f52-fbeb-437d-89a4-cdae39086298 {"id":"6a508f52-fbeb-437d-89a4-cdae39086298","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899208} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087df278e8 6a508f52-fbeb-437d-89a4-cdae39086298 /tidb/cdc/default/default/upstream/7489990016759345567 {"id":7489990016759345567,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6a508f52-fbeb-437d-89a4-cdae39086298 {"id":"6a508f52-fbeb-437d-89a4-cdae39086298","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899208} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087df278e8 6a508f52-fbeb-437d-89a4-cdae39086298 /tidb/cdc/default/default/upstream/7489990016759345567 {"id":7489990016759345567,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + config_path=conf/changefeed.toml + SINK_URI='mysql://root@127.0.0.1:3306/?max-txn-row=1' + run_cdc_cli changefeed create --start-ts=457152713676554241 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/synced_status/conf/changefeed.toml + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.17525.out cli changefeed create --start-ts=457152713676554241 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/synced_status/conf/changefeed.toml [2025/04/06 08:26:51.833 +08:00] [INFO] [main.go:221] ["0 insert success: 600"] [2025/04/06 08:26:51.836 +08:00] [INFO] [main.go:235] ["0 delete success: 300"] [2025/04/06 08:26:51.872 +08:00] [INFO] [main.go:235] ["1 delete success: 300"] [2025/04/06 08:26:51.873 +08:00] [INFO] [main.go:221] ["1 insert success: 600"] Create changefeed successfully! ID: test-1 Info: {"upstream_id":7489990016759345567,"namespace":"default","id":"test-1","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2025-04-06T08:26:51.819321908+08:00","start_ts":457152713676554241,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":120,"checkpoint_interval":20}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457152713676554241,"checkpoint_ts":457152713676554241,"checkpoint_time":"2025-04-06 08:26:46.835"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... running remove_changefeed [2025/04/06 08:26:52.098 +08:00] [INFO] [main.go:221] ["0 insert success: 700"] [2025/04/06 08:26:52.126 +08:00] [INFO] [main.go:221] ["1 insert success: 700"] \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } pass test: remove changefeed [Pipeline] // timeout [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-periodics_cdc_integration_mysql_test-234/tiflow-cdc already exists) [2025/04/06 08:26:52.363 +08:00] [INFO] [main.go:221] ["0 insert success: 800"] [2025/04/06 08:26:52.366 +08:00] [INFO] [main.go:235] ["0 delete success: 400"] [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [2025/04/06 08:26:52.452 +08:00] [INFO] [main.go:235] ["1 delete success: 400"] [2025/04/06 08:26:52.454 +08:00] [INFO] [main.go:221] ["1 insert success: 800"] [2025/04/06 08:26:52.642 +08:00] [INFO] [main.go:221] ["0 insert success: 900"] [2025/04/06 08:26:52.677 +08:00] [INFO] [main.go:221] ["1 insert success: 900"] [Pipeline] // stage [Pipeline] } curl http://10.233.76.109:2379/ % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 19 100 19 0 0 16239 0 --:--:-- --:--:-- --:--:-- 19000 404 page not found run task successfully + echo started proxy pid: 9345 started proxy pid: 9345 + echo started proxy at port: 13444 started proxy at port: 13444 + cd /tmp/tidb_cdc_test/http_proxies ++ run_cdc_cli_tso_query 10.233.76.109 2379 + pd_host=10.233.76.109 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://10.233.76.109:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.http_proxies.cli.13291.out cli tso query --pd=http://10.233.76.109:2379 check diff successfully [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [2025/04/06 08:26:52.980 +08:00] [INFO] [main.go:221] ["0 insert success: 1000"] [2025/04/06 08:26:53.026 +08:00] [INFO] [main.go:235] ["0 delete success: 500"] [2025/04/06 08:26:53.061 +08:00] [INFO] [main.go:235] ["1 delete success: 500"] [2025/04/06 08:26:53.062 +08:00] [INFO] [main.go:221] ["1 insert success: 1000"] [2025/04/06 08:26:53.238 +08:00] [INFO] [main.go:221] ["0 insert success: 1100"] [Pipeline] // withEnv [Pipeline] } wait process cdc.test exit for 1-th time... [Pipeline] // stage [Pipeline] } + set +x ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 221 100 221 0 0 5752 0 --:--:-- --:--:-- --:--:-- 5815 + synced_status='{"synced":true,"sink_checkpoint_ts":"2025-04-06 08:26:53.134","puller_resolved_ts":"1970-01-01 08:00:00.000","last_synced_ts":"1970-01-01 08:00:00.000","now_ts":"2025-04-06 08:26:53.000","info":"Data syncing is finished"}' ++ echo '{"synced":true,"sink_checkpoint_ts":"2025-04-06' '08:26:53.134","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2025-04-06' '08:26:53.000","info":"Data' syncing is 'finished"}' ++ jq .synced + status=true ++ echo '{"synced":true,"sink_checkpoint_ts":"2025-04-06' '08:26:53.134","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2025-04-06' '08:26:53.000","info":"Data' syncing is 'finished"}' ++ jq -r .sink_checkpoint_ts + sink_checkpoint_ts='2025-04-06 08:26:53.134' ++ echo '{"synced":true,"sink_checkpoint_ts":"2025-04-06' '08:26:53.134","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2025-04-06' '08:26:53.000","info":"Data' syncing is 'finished"}' ++ jq -r .puller_resolved_ts + puller_resolved_ts='1970-01-01 08:00:00.000' ++ echo '{"synced":true,"sink_checkpoint_ts":"2025-04-06' '08:26:53.134","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2025-04-06' '08:26:53.000","info":"Data' syncing is 'finished"}' ++ jq -r .last_synced_ts [2025/04/06 08:26:53.276 +08:00] [INFO] [main.go:221] ["1 insert success: 1100"] [2025/04/06 08:26:53.461 +08:00] [INFO] [main.go:221] ["0 insert success: 1200"] [2025/04/06 08:26:53.464 +08:00] [INFO] [main.go:235] ["0 delete success: 600"] + last_synced_ts='1970-01-01 08:00:00.000' + '[' true '!=' true ']' + '[' '1970-01-01 08:00:00.000' '!=' '1970-01-01 08:00:00.000' ']' + '[' '1970-01-01 08:00:00.000' '!=' '1970-01-01 08:00:00.000' ']' ++ date '+%Y-%m-%d %H:%M:%S' + current='2025-04-06 08:26:53' + echo 'sink_checkpoint_ts is 2025-04-06' 08:26:53.134 sink_checkpoint_ts is 2025-04-06 08:26:53.134 ++ date -d '2025-04-06 08:26:53.134' +%s + checkpoint_timestamp=1743899213 ++ date -d '2025-04-06 08:26:53' +%s + current_timestamp=1743899213 + '[' 0 -gt 300 ']' + run_sql 'USE TEST;Create table t1(a int primary key, b int);insert into t1 values(1,2);insert into t1 values(2,3);' + check_table_exists test.t1 127.0.0.1 3306 table test.t1 not exists for 1-th check, retry later running resign_owner pass test: resign owner [2025/04/06 08:26:53.537 +08:00] [INFO] [main.go:235] ["1 delete success: 600"] [2025/04/06 08:26:53.538 +08:00] [INFO] [main.go:221] ["1 insert success: 1200"] wait process cdc.test exit for 2-th time... [2025/04/06 08:26:53.829 +08:00] [INFO] [main.go:221] ["0 insert success: 1300"] [2025/04/06 08:26:53.868 +08:00] [INFO] [main.go:221] ["1 insert success: 1300"] + set +x failed proxying s2c rpc error: code = Canceled desc = context canceled failed proxying s2c rpc error: code = Canceled desc = context canceled + tso='457152715212980225 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 457152715212980225 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + start_ts=457152715212980225 + echo 'query start ts: 457152715212980225' query start ts: 457152715212980225 + run_cdc_server --workdir /tmp/tidb_cdc_test/http_proxies --binary cdc.test [Sun Apr 6 08:26:54 CST 2025] <<<<<< START cdc server in http_proxies case >>>>>> cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Sun Apr 6 08:26:54 CST 2025] <<<<<< run test case autorandom success! >>>>>> [2025/04/06 08:26:54.044 +08:00] [INFO] [main.go:221] ["0 insert success: 1400"] [2025/04/06 08:26:54.046 +08:00] [INFO] [main.go:235] ["0 delete success: 700"] [2025/04/06 08:26:54.081 +08:00] [INFO] [main.go:235] ["1 delete success: 700"] [2025/04/06 08:26:54.083 +08:00] [INFO] [main.go:221] ["1 insert success: 1400"] + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.http_proxies.1333213334.out server --log-file /tmp/tidb_cdc_test/http_proxies/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/http_proxies/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to proxy 127.0.0.1 port 13444 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 13444 (#0) * Server auth using Basic with user 'ticdc' > GET http://127.0.0.1:8300/debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > Proxy-Connection: Keep-Alive > < HTTP/1.1 502 Bad Gateway < Date: Sun, 06 Apr 2025 00:26:54 GMT < Content-Length: 0 < * Connection #0 to host 127.0.0.1 left intact + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2025/04/06 08:26:54.365 +08:00] [INFO] [main.go:221] ["0 insert success: 1500"] [2025/04/06 08:26:54.406 +08:00] [INFO] [main.go:221] ["1 insert success: 1500"] running get_tso failed proxying s2c rpc error: code = Canceled desc = context canceled failed proxying s2c rpc error: code = Canceled desc = context canceled failed proxying s2c rpc error: code = Canceled desc = context canceled failed proxying s2c rpc error: code = Canceled desc = context canceled + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 06 Apr 2025 00:26:54 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/e4e316b2-866d-4639-852a-40150af2b5e8 {UpstreamID:7489989903191531286 Namespace:default ID:e4e316b2-866d-4639-852a-40150af2b5e8 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-06 08:26:23.426425674 +0800 CST StartTs:457152707514335233 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc004110ab0 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457152707527442440} {CheckpointTs:457152710542098451 MinTableBarrierTs:457152715535941637 AdminJobType:noop} span: {tableID:123, startKey:7480000000000000ff7b5f720000000000fa, endKey:7480000000000000ff7b5f730000000000fa}, resolvedTs: 457152710542098451, checkpointTs: 457152710542098451, state: Replicating span: {tableID:120, startKey:7480000000000000ff785f720000000000fa, endKey:7480000000000000ff785f730000000000fa}, resolvedTs: 457152710542098451, checkpointTs: 457152710542098451, state: Replicating span: {tableID:119, startKey:7480000000000000ff775f720000000000fa, endKey:7480000000000000ff775f730000000000fa}, resolvedTs: 457152710542098451, checkpointTs: 457152710542098451, state: Replicating span: {tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}, resolvedTs: 457152710542098451, checkpointTs: 457152710542098451, state: Replicating span: {tableID:116, startKey:7480000000000000ff745f720000000000fa, endKey:7480000000000000ff745f730000000000fa}, resolvedTs: 457152710542098451, checkpointTs: 457152710542098451, state: Replicating span: {tableID:124, startKey:7480000000000000ff7c5f720000000000fa, endKey:7480000000000000ff7c5f730000000000fa}, resolvedTs: 457152710542098451, checkpointTs: 457152710542098451, state: Replicating span: {tableID:126, startKey:7480000000000000ff7e5f720000000000fa, endKey:7480000000000000ff7e5f730000000000fa}, resolvedTs: 457152710542098451, checkpointTs: 457152710542098451, state: Replicating span: {tableID:121, startKey:7480000000000000ff795f720000000000fa, endKey:7480000000000000ff795f730000000000fa}, resolvedTs: 457152710542098451, checkpointTs: 457152710542098451, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5cb81288-da08-4464-969f-4510637545a8 {"id":"5cb81288-da08-4464-969f-4510637545a8","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899211} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087d8dae48 5cb81288-da08-4464-969f-4510637545a8 /tidb/cdc/default/default/changefeed/info/e4e316b2-866d-4639-852a-40150af2b5e8 {"upstream-id":7489989903191531286,"namespace":"default","changefeed-id":"e4e316b2-866d-4639-852a-40150af2b5e8","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-06T08:26:23.426425674+08:00","start-ts":457152707514335233,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"file:///tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/redo","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457152707527442440} /tidb/cdc/default/default/changefeed/status/e4e316b2-866d-4639-852a-40150af2b5e8 {"checkpoint-ts":457152710542098451,"min-table-barrier-ts":457152715535941637,"admin-job-type":0} /tidb/cdc/default/default/task/position/5cb81288-da08-4464-969f-4510637545a8/e4e316b2-866d-4639-852a-40150af2b5e8 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489989903191531286 {"id":7489989903191531286,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/e4e316b2-866d-4639-852a-40150af2b5e8 {UpstreamID:7489989903191531286 Namespace:default ID:e4e316b2-866d-4639-852a-40150af2b5e8 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-06 08:26:23.426425674 +0800 CST StartTs:457152707514335233 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc004110ab0 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457152707527442440} {CheckpointTs:457152710542098451 MinTableBarrierTs:457152715535941637 AdminJobType:noop} span: {tableID:123, startKey:7480000000000000ff7b5f720000000000fa, endKey:7480000000000000ff7b5f730000000000fa}, resolvedTs: 457152710542098451, checkpointTs: 457152710542098451, state: Replicating span: {tableID:120, startKey:7480000000000000ff785f720000000000fa, endKey:7480000000000000ff785f730000000000fa}, resolvedTs: 457152710542098451, checkpointTs: 457152710542098451, state: Replicating span: {tableID:119, startKey:7480000000000000ff775f720000000000fa, endKey:7480000000000000ff775f730000000000fa}, resolvedTs: 457152710542098451, checkpointTs: 457152710542098451, state: Replicating span: {tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}, resolvedTs: 457152710542098451, checkpointTs: 457152710542098451, state: Replicating span: {tableID:116, startKey:7480000000000000ff745f720000000000fa, endKey:7480000000000000ff745f730000000000fa}, resolvedTs: 457152710542098451, checkpointTs: 457152710542098451, state: Replicating span: {tableID:124, startKey:7480000000000000ff7c5f720000000000fa, endKey:7480000000000000ff7c5f730000000000fa}, resolvedTs: 457152710542098451, checkpointTs: 457152710542098451, state: Replicating span: {tableID:126, startKey:7480000000000000ff7e5f720000000000fa, endKey:7480000000000000ff7e5f730000000000fa}, resolvedTs: 457152710542098451, checkpointTs: 457152710542098451, state: Replicating span: {tableID:121, startKey:7480000000000000ff795f720000000000fa, endKey:7480000000000000ff795f730000000000fa}, resolvedTs: 457152710542098451, checkpointTs: 457152710542098451, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5cb81288-da08-4464-969f-4510637545a8 {"id":"5cb81288-da08-4464-969f-4510637545a8","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899211} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087d8dae48 5cb81288-da08-4464-969f-4510637545a8 /tidb/cdc/default/default/changefeed/info/e4e316b2-866d-4639-852a-40150af2b5e8 {"upstream-id":7489989903191531286,"namespace":"default","changefeed-id":"e4e316b2-866d-4639-852a-40150af2b5e8","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-06T08:26:23.426425674+08:00","start-ts":457152707514335233,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"file:///tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/redo","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457152707527442440} /tidb/cdc/default/default/changefeed/status/e4e316b2-866d-4639-852a-40150af2b5e8 {"checkpoint-ts":457152710542098451,"min-table-barrier-ts":457152715535941637,"admin-job-type":0} /tidb/cdc/default/default/task/position/5cb81288-da08-4464-969f-4510637545a8/e4e316b2-866d-4639-852a-40150af2b5e8 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489989903191531286 {"id":7489989903191531286,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/e4e316b2-866d-4639-852a-40150af2b5e8 {UpstreamID:7489989903191531286 Namespace:default ID:e4e316b2-866d-4639-852a-40150af2b5e8 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-06 08:26:23.426425674 +0800 CST StartTs:457152707514335233 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc004110ab0 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457152707527442440} {CheckpointTs:457152710542098451 MinTableBarrierTs:457152715535941637 AdminJobType:noop} span: {tableID:123, startKey:7480000000000000ff7b5f720000000000fa, endKey:7480000000000000ff7b5f730000000000fa}, resolvedTs: 457152710542098451, checkpointTs: 457152710542098451, state: Replicating span: {tableID:120, startKey:7480000000000000ff785f720000000000fa, endKey:7480000000000000ff785f730000000000fa}, resolvedTs: 457152710542098451, checkpointTs: 457152710542098451, state: Replicating span: {tableID:119, startKey:7480000000000000ff775f720000000000fa, endKey:7480000000000000ff775f730000000000fa}, resolvedTs: 457152710542098451, checkpointTs: 457152710542098451, state: Replicating span: {tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}, resolvedTs: 457152710542098451, checkpointTs: 457152710542098451, state: Replicating span: {tableID:116, startKey:7480000000000000ff745f720000000000fa, endKey:7480000000000000ff745f730000000000fa}, resolvedTs: 457152710542098451, checkpointTs: 457152710542098451, state: Replicating span: {tableID:124, startKey:7480000000000000ff7c5f720000000000fa, endKey:7480000000000000ff7c5f730000000000fa}, resolvedTs: 457152710542098451, checkpointTs: 457152710542098451, state: Replicating span: {tableID:126, startKey:7480000000000000ff7e5f720000000000fa, endKey:7480000000000000ff7e5f730000000000fa}, resolvedTs: 457152710542098451, checkpointTs: 457152710542098451, state: Replicating span: {tableID:121, startKey:7480000000000000ff795f720000000000fa, endKey:7480000000000000ff795f730000000000fa}, resolvedTs: 457152710542098451, checkpointTs: 457152710542098451, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5cb81288-da08-4464-969f-4510637545a8 {"id":"5cb81288-da08-4464-969f-4510637545a8","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899211} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087d8dae48 5cb81288-da08-4464-969f-4510637545a8 /tidb/cdc/default/default/changefeed/info/e4e316b2-866d-4639-852a-40150af2b5e8 {"upstream-id":7489989903191531286,"namespace":"default","changefeed-id":"e4e316b2-866d-4639-852a-40150af2b5e8","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-06T08:26:23.426425674+08:00","start-ts":457152707514335233,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"file:///tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/redo","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457152707527442440} /tidb/cdc/default/default/changefeed/status/e4e316b2-866d-4639-852a-40150af2b5e8 {"checkpoint-ts":457152710542098451,"min-table-barrier-ts":457152715535941637,"admin-job-type":0} /tidb/cdc/default/default/task/position/5cb81288-da08-4464-969f-4510637545a8/e4e316b2-866d-4639-852a-40150af2b5e8 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489989903191531286 {"id":7489989903191531286,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x [2025/04/06 08:26:54.577 +08:00] [INFO] [main.go:221] ["0 insert success: 1600"] [2025/04/06 08:26:54.579 +08:00] [INFO] [main.go:235] ["0 delete success: 800"] [2025/04/06 08:26:54.613 +08:00] [INFO] [main.go:235] ["1 delete success: 800"] [2025/04/06 08:26:54.614 +08:00] [INFO] [main.go:221] ["1 insert success: 1600"] pass test: get tso table test.t1 exists + sleep 5 running verify_table pass test: verify table running create_changefeed_v2 pass test: create changefeed v2 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to proxy 127.0.0.1 port 13444 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 13444 (#0) * Server auth using Basic with user 'ticdc' > GET http://127.0.0.1:8300/debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > Proxy-Connection: Keep-Alive > < HTTP/1.1 200 OK < Content-Length: 824 < Content-Type: text/plain; charset=utf-8 < Date: Sun, 06 Apr 2025 00:26:57 GMT < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e1cfb723-dae8-4fba-9303-c104505dae66 {"id":"e1cfb723-dae8-4fba-9303-c104505dae66","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899214} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/330a96087da72385 e1cfb723-dae8-4fba-9303-c104505dae66 /tidb/cdc/default/default/upstream/7489989926198349211 {"id":7489989926198349211,"pd-endpoints":"http://127.0.0.1:2379,http://10.233.76.109:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e1cfb723-dae8-4fba-9303-c104505dae66 {"id":"e1cfb723-dae8-4fba-9303-c104505dae66","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899214} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/330a96087da72385 e1cfb723-dae8-4fba-9303-c104505dae66 /tidb/cdc/default/default/upstream/7489989926198349211 {"id":7489989926198349211,"pd-endpoints":"http://127.0.0.1:2379,http://10.233.76.109:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e1cfb723-dae8-4fba-9303-c104505dae66 {"id":"e1cfb723-dae8-4fba-9303-c104505dae66","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899214} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/330a96087da72385 e1cfb723-dae8-4fba-9303-c104505dae66 /tidb/cdc/default/default/upstream/7489989926198349211 {"id":7489989926198349211,"pd-endpoints":"http://127.0.0.1:2379,http://10.233.76.109:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + echo started cdc server successfully started cdc server successfully + SINK_URI=blackhole:/// + run_cdc_cli changefeed create --start-ts=457152715212980225 --sink-uri=blackhole:/// + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.http_proxies.cli.13395.out cli changefeed create --start-ts=457152715212980225 --sink-uri=blackhole:/// Create changefeed successfully! ID: a6eecff3-c3a0-44ee-b0c0-a7628a871a50 Info: {"upstream_id":7489989926198349211,"namespace":"default","id":"a6eecff3-c3a0-44ee-b0c0-a7628a871a50","sink_uri":"blackhole:///","create_time":"2025-04-06T08:26:57.621144003+08:00","start_ts":457152715212980225,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457152715212980225,"checkpoint_ts":457152715212980225,"checkpoint_time":"2025-04-06 08:26:52.696"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... running delete_changefeed_v2 pass test: remove changefeed v2 + set +x + sleep 5 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/changefeed_dup_error_restart/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... running unsafe_apis status code 200 pass test: show metadata status code 200 status code 200 pass test: delete service_gc_safepoint status code 500 pass test: resolve lock ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 243 100 243 0 0 4900 0 --:--:-- --:--:-- --:--:-- 4959 + synced_status='{"synced":false,"sink_checkpoint_ts":"2025-04-06 08:27:00.185","puller_resolved_ts":"2025-04-06 08:26:53.435","last_synced_ts":"2025-04-06 08:26:53.435","now_ts":"2025-04-06 08:27:00.000","info":"The data syncing is not finished, please wait"}' ++ echo '{"synced":false,"sink_checkpoint_ts":"2025-04-06' '08:27:00.185","puller_resolved_ts":"2025-04-06' '08:26:53.435","last_synced_ts":"2025-04-06' '08:26:53.435","now_ts":"2025-04-06' '08:27:00.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq .synced + status=false + '[' false '!=' false ']' ++ echo '{"synced":false,"sink_checkpoint_ts":"2025-04-06' '08:27:00.185","puller_resolved_ts":"2025-04-06' '08:26:53.435","last_synced_ts":"2025-04-06' '08:26:53.435","now_ts":"2025-04-06' '08:27:00.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq -r .info + info='The data syncing is not finished, please wait' + '[' 'The data syncing is not finished, please wait' '!=' 'The data syncing is not finished, please wait' ']' + sleep 130 [2025/04/06 08:27:00.695 +08:00] [INFO] [main.go:86] ["running ddl test: 5 addDropIndexDDL"] [2025/04/06 08:27:00.930 +08:00] [INFO] [main.go:221] ["0 insert success: 100"] [2025/04/06 08:27:00.931 +08:00] [INFO] [main.go:221] ["1 insert success: 100"] [2025/04/06 08:27:01.252 +08:00] [INFO] [main.go:235] ["0 delete success: 100"] [2025/04/06 08:27:01.253 +08:00] [INFO] [main.go:221] ["0 insert success: 200"] [2025/04/06 08:27:01.255 +08:00] [INFO] [main.go:221] ["1 insert success: 200"] [2025/04/06 08:27:01.257 +08:00] [INFO] [main.go:235] ["1 delete success: 100"] [2025/04/06 08:27:01.437 +08:00] [INFO] [main.go:221] ["0 insert success: 300"] [2025/04/06 08:27:01.442 +08:00] [INFO] [main.go:221] ["1 insert success: 300"] [2025/04/06 08:27:01.663 +08:00] [INFO] [main.go:235] ["0 delete success: 200"] [2025/04/06 08:27:01.664 +08:00] [INFO] [main.go:221] ["0 insert success: 400"] [2025/04/06 08:27:01.729 +08:00] [INFO] [main.go:221] ["1 insert success: 400"] [2025/04/06 08:27:01.732 +08:00] [INFO] [main.go:235] ["1 delete success: 200"] [2025/04/06 08:27:01.937 +08:00] [INFO] [main.go:221] ["0 insert success: 500"] [2025/04/06 08:27:01.944 +08:00] [INFO] [main.go:221] ["1 insert success: 500"] wait process cdc.test exit for 1-th time... start tidb cluster in /tmp/tidb_cdc_test/changefeed_dup_error_restart Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... [2025/04/06 08:27:02.248 +08:00] [INFO] [main.go:235] ["0 delete success: 300"] [2025/04/06 08:27:02.249 +08:00] [INFO] [main.go:221] ["0 insert success: 600"] [2025/04/06 08:27:02.256 +08:00] [INFO] [main.go:221] ["1 insert success: 600"] [2025/04/06 08:27:02.259 +08:00] [INFO] [main.go:235] ["1 delete success: 300"] wait process cdc.test exit for 2-th time... [2025/04/06 08:27:02.594 +08:00] [INFO] [main.go:221] ["0 insert success: 700"] [2025/04/06 08:27:02.594 +08:00] [INFO] [main.go:221] ["1 insert success: 700"] wait process cdc.test exit for 3-th time... [2025/04/06 08:27:03.002 +08:00] [INFO] [main.go:221] ["1 insert success: 800"] [2025/04/06 08:27:03.006 +08:00] [INFO] [main.go:235] ["1 delete success: 400"] [2025/04/06 08:27:03.029 +08:00] [INFO] [main.go:235] ["0 delete success: 400"] [2025/04/06 08:27:03.031 +08:00] [INFO] [main.go:221] ["0 insert success: 800"] cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Sun Apr 6 08:27:03 CST 2025] <<<<<< run test case http_api_tls_with_user_auth success! >>>>>> + stop_tidb_cluster [2025/04/06 08:27:03.467 +08:00] [INFO] [main.go:221] ["1 insert success: 900"] [2025/04/06 08:27:03.579 +08:00] [INFO] [main.go:221] ["0 insert success: 900"] [2025/04/06 08:27:03.815 +08:00] [INFO] [main.go:221] ["1 insert success: 1000"] [2025/04/06 08:27:03.818 +08:00] [INFO] [main.go:235] ["1 delete success: 500"] =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/generate_column/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [2025/04/06 08:27:04.056 +08:00] [INFO] [main.go:235] ["0 delete success: 500"] [2025/04/06 08:27:04.057 +08:00] [INFO] [main.go:221] ["0 insert success: 1000"] [2025/04/06 08:27:04.134 +08:00] [INFO] [main.go:221] ["1 insert success: 1100"] [2025/04/06 08:27:04.353 +08:00] [INFO] [main.go:221] ["1 insert success: 1200"] [2025/04/06 08:27:04.355 +08:00] [INFO] [main.go:235] ["1 delete success: 600"] [2025/04/06 08:27:04.382 +08:00] [INFO] [main.go:221] ["0 insert success: 1100"] + check + services=($(cat $WORK_DIR/test_proxy.log | xargs -L1 dirname | sort | uniq)) ++ cat /tmp/tidb_cdc_test/http_proxies/test_proxy.log ++ xargs -L1 dirname ++ sort ++ uniq + service_type_count=35 + echo 'captured services: ' captured services: + echo . /cdcpb.ChangeData /grpc.health.v1.Health /pdpb.PD /resource_manager.ResourceManager /tikvpb.Tikv '[2025/04' '[metadata=MD{:authority=[10.233.76.109:2379],' 'user-agent=[grpc-go/1.64.0],' 'grpc-accept-encoding=[gzip],' 'content-type=[application' '[metadata=MD{content-type=[application/grpc],' ':authority=[10.233.76.109:20160],' 'grpc-accept-encoding=[gzip],' 'features=[stream-multiplexing],' 'user-agent=[grpc-go' '[metadata=MD{content-type=[application/grpc],' ':authority=[10.233.76.109:2379],' 'user-agent=[grpc-go' '[metadata=MD{features=[stream-multiplexing],' 'user-agent=[grpc-go/1.64.0],' 'content-type=[application' '[metadata=MD{grpc-accept-encoding=[gzip],' 'features=[stream-multiplexing],' 'user-agent=[grpc-go/1.64.0],' 'content-type=[application' '[metadata=MD{user-agent=[grpc-go/1.64.0],' 'content-type=[application' '[metadata=MD{user-agent=[grpc-go/1.64.0],' 'grpc-accept-encoding=[gzip],' 'content-type=[application' '[method=/cdcpb.ChangeData' '[method=/pdpb.PD' '[method=/resource_manager.ResourceManager' '[stack=main.intercept\n\t/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/http_proxies/run-proxy.go:56\ngithub.com/bradleyjkemp/grpc-tools/grpc-proxy.recoverWrapper.func1\n\t/go/pkg/mod/github.com/bradleyjkemp/grpc-tools@v0.2.5/grpc-proxy/config.go:52\ngoogle.golang.org/grpc.(*Server).processStreamingRPC\n\t/go/pkg/mod/google.golang.org/grpc@v1.64.0/server.go:1680\ngoogle.golang.org/grpc.(*Server).handleStream\n\t/go/pkg/mod/google.golang.org/grpc@v1.64.0/server.go:1800\ngoogle.golang.org/grpc.(*Server).serveStreams.func2.1\n\t/go/pkg/mod/google.golang.org/grpc@v1.64.0' . /cdcpb.ChangeData /grpc.health.v1.Health /pdpb.PD /resource_manager.ResourceManager /tikvpb.Tikv [2025/04 [metadata=MD{:authority=[10.233.76.109:2379], user-agent=[grpc-go/1.64.0], grpc-accept-encoding=[gzip], content-type=[application [metadata=MD{content-type=[application/grpc], :authority=[10.233.76.109:20160], grpc-accept-encoding=[gzip], features=[stream-multiplexing], user-agent=[grpc-go [metadata=MD{content-type=[application/grpc], :authority=[10.233.76.109:2379], user-agent=[grpc-go [metadata=MD{features=[stream-multiplexing], user-agent=[grpc-go/1.64.0], content-type=[application [metadata=MD{grpc-accept-encoding=[gzip], features=[stream-multiplexing], user-agent=[grpc-go/1.64.0], content-type=[application [metadata=MD{user-agent=[grpc-go/1.64.0], content-type=[application [metadata=MD{user-agent=[grpc-go/1.64.0], grpc-accept-encoding=[gzip], content-type=[application [method=/cdcpb.ChangeData [method=/pdpb.PD [method=/resource_manager.ResourceManager [stack=main.intercept\n\t/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/http_proxies/run-proxy.go:56\ngithub.com/bradleyjkemp/grpc-tools/grpc-proxy.recoverWrapper.func1\n\t/go/pkg/mod/github.com/bradleyjkemp/grpc-tools@v0.2.5/grpc-proxy/config.go:52\ngoogle.golang.org/grpc.(*Server).processStreamingRPC\n\t/go/pkg/mod/google.golang.org/grpc@v1.64.0/server.go:1680\ngoogle.golang.org/grpc.(*Server).handleStream\n\t/go/pkg/mod/google.golang.org/grpc@v1.64.0/server.go:1800\ngoogle.golang.org/grpc.(*Server).serveStreams.func2.1\n\t/go/pkg/mod/google.golang.org/grpc@v1.64.0 + '[' 35 -ge 2 ']' + check_logs /tmp/tidb_cdc_test/http_proxies ++ date + echo '[Sun Apr 6 08:27:04 CST 2025] <<<<<< run test case http_proxies success! >>>>>>' [Sun Apr 6 08:27:04 CST 2025] <<<<<< run test case http_proxies success! >>>>>> + stop_tidb_cluster failed proxying s2c rpc error: code = Canceled desc = context canceled failed proxying s2c rpc error: code = Canceled desc = context canceled failed proxying s2c rpc error: code = Canceled desc = context canceled failed proxying s2c rpc error: code = Canceled desc = context canceled failed proxying s2c rpc error: code = Canceled desc = context canceled failed proxying s2c rpc error: code = Canceled desc = context canceled [2025/04/06 08:27:04.577 +08:00] [INFO] [main.go:221] ["1 insert success: 1300"] [2025/04/06 08:27:04.697 +08:00] [INFO] [main.go:235] ["0 delete success: 600"] [2025/04/06 08:27:04.698 +08:00] [INFO] [main.go:221] ["0 insert success: 1200"] [2025/04/06 08:27:04.799 +08:00] [INFO] [main.go:221] ["1 insert success: 1400"] [2025/04/06 08:27:04.801 +08:00] [INFO] [main.go:235] ["1 delete success: 700"] [2025/04/06 08:27:05.112 +08:00] [INFO] [main.go:221] ["0 insert success: 1300"] [2025/04/06 08:27:05.158 +08:00] [INFO] [main.go:221] ["1 insert success: 1500"] [2025/04/06 08:27:05.552 +08:00] [INFO] [main.go:235] ["0 delete success: 700"] [2025/04/06 08:27:05.554 +08:00] [INFO] [main.go:221] ["0 insert success: 1400"] [2025/04/06 08:27:05.562 +08:00] [INFO] [main.go:221] ["1 insert success: 1600"] [2025/04/06 08:27:05.567 +08:00] [INFO] [main.go:235] ["1 delete success: 800"] Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release start tidb cluster in /tmp/tidb_cdc_test/generate_column Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } [Pipeline] // timeout [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-periodics_cdc_integration_mysql_test-234/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489990138194206589 TiDB Cluster ID. tikv_gc_leader_uuid 65821f9f8e00013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-9-92vm1, pid:18264, start at 2025-04-06 08:27:09.784390686 +0800 CST m=+2.031874615 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:29:09.834 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:27:09.802 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:17:09.802 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489990138194206589 TiDB Cluster ID. tikv_gc_leader_uuid 65821f9f8e00013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-9-92vm1, pid:18264, start at 2025-04-06 08:27:09.784390686 +0800 CST m=+2.031874615 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:29:09.834 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:27:09.802 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:17:09.802 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489990135750469435 TiDB Cluster ID. tikv_gc_leader_uuid 65821f9fbf8000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-9-92vm1, pid:18339, start at 2025-04-06 08:27:09.968702393 +0800 CST m=+2.134829169 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:29:09.976 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:27:09.950 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:17:09.950 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/changefeed_dup_error_restart/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/changefeed_dup_error_restart/tiflash/log/error.log arg matches is ArgMatches { args: {"log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/changefeed_dup_error_restart/tiflash/log/proxy.log"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/changefeed_dup_error_restart/tiflash-proxy.toml"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/changefeed_dup_error_restart/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [2025/04/06 08:27:13.159 +08:00] [INFO] [main.go:78] ["runDDLTest take 1m6.569378267s"] table mark.finish_mark_0 exists table mark.finish_mark_1 exists ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_dup_error_restart.cli.19583.out cli tso query --pd=http://127.0.0.1:2379 table mark.finish_mark_2 exists table mark.finish_mark_3 exists table mark.finish_mark_4 exists table mark.finish_mark_5 exists table mark.finish_mark not exists for 1-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/cli_tls_with_auth/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489990153678648305 TiDB Cluster ID. tikv_gc_leader_uuid 65821fa36fc000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-j-vgmw9, pid:30849, start at 2025-04-06 08:27:13.742830205 +0800 CST m=+2.191540919 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:29:13.751 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:27:13.752 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:17:13.752 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489990153678648305 TiDB Cluster ID. tikv_gc_leader_uuid 65821fa36fc000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-j-vgmw9, pid:30849, start at 2025-04-06 08:27:13.742830205 +0800 CST m=+2.191540919 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:29:13.751 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:27:13.752 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:17:13.752 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489990153130731001 TiDB Cluster ID. tikv_gc_leader_uuid 65821fa39a80006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-j-vgmw9, pid:30918, start at 2025-04-06 08:27:13.934704226 +0800 CST m=+2.303892261 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:29:13.942 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:27:13.949 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:17:13.949 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/generate_column/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/generate_column/tiflash/log/error.log arg matches is ArgMatches { args: {"config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/generate_column/tiflash-proxy.toml"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/generate_column/tiflash/log/proxy.log"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/generate_column/tiflash/db/proxy"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + set +x + tso='457152720916447233 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 457152720916447233 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x ***************** properties ***************** "requestdistribution"="uniform" "dotransactions"="false" "mysql.port"="4000" "threadcount"="2" "mysql.user"="root" "readallfields"="true" "workload"="core" "readproportion"="0" "mysql.host"="127.0.0.1" "operationcount"="0" "scanproportion"="0" "insertproportion"="0" "mysql.db"="changefeed_dup_error_restart" "recordcount"="10" "updateproportion"="0" ********************************************** Run finished, takes 17.182712ms INSERT - Takes(s): 0.0, Count: 10, OPS: 750.1, Avg(us): 3207, Min(us): 952, Max(us): 8108, 95th(us): 9000, 99th(us): 9000 [Sun Apr 6 08:27:15 CST 2025] <<<<<< START cdc server in changefeed_dup_error_restart case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sink/dmlsink/txn/mysql/MySQLDuplicateEntryError=5%return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_dup_error_restart.1964719649.out server --log-file /tmp/tidb_cdc_test/changefeed_dup_error_restart/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_dup_error_restart/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table mark.finish_mark exists check diff successfully check_redo_resolved_ts e4e316b2-866d-4639-852a-40150af2b5e8 457152721264050179 file:///tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/redo /tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/cdc_data/redo/e4e316b2-866d-4639-852a-40150af2b5e8/meta global resolved ts 457152720503570438 not forward to 457152721264050179 run task failed 1-th time, retry later wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... start tidb cluster in /tmp/tidb_cdc_test/cli_tls_with_auth Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.generate_column.cli.32242.out cli tso query --pd=http://127.0.0.1:2379 cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit check_redo_resolved_ts e4e316b2-866d-4639-852a-40150af2b5e8 457152721264050179 file:///tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/redo /tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/cdc_data/redo/e4e316b2-866d-4639-852a-40150af2b5e8/meta global resolved ts 457152721027858439 not forward to 457152721264050179 run task failed 2-th time, retry later + stop_proxy + kill 9345 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/sequence/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 06 Apr 2025 00:27:19 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/cc6cd067-e9eb-4ffa-bcee-f5eafd82205a {"id":"cc6cd067-e9eb-4ffa-bcee-f5eafd82205a","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899236} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087e6082f4 cc6cd067-e9eb-4ffa-bcee-f5eafd82205a /tidb/cdc/default/default/upstream/7489990138194206589 {"id":7489990138194206589,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/cc6cd067-e9eb-4ffa-bcee-f5eafd82205a {"id":"cc6cd067-e9eb-4ffa-bcee-f5eafd82205a","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899236} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087e6082f4 cc6cd067-e9eb-4ffa-bcee-f5eafd82205a /tidb/cdc/default/default/upstream/7489990138194206589 {"id":7489990138194206589,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/cc6cd067-e9eb-4ffa-bcee-f5eafd82205a {"id":"cc6cd067-e9eb-4ffa-bcee-f5eafd82205a","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899236} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087e6082f4 cc6cd067-e9eb-4ffa-bcee-f5eafd82205a /tidb/cdc/default/default/upstream/7489990138194206589 {"id":7489990138194206589,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_dup_error_restart.cli.19706.out cli changefeed create --start-ts=457152720916447233 '--sink-uri=mysql://normal:123456@127.0.0.1:3306/?max-txn-row=1' [2025/04/06 08:27:19.470 +08:00] [DEBUG] [pool.rs:146] ["scheduling task [split check worker] Split Check Task for 178, start_key: None, end_key: None, auto_split: true"] [thread_id=91] [2025/04/06 08:27:19.470 +08:00] [DEBUG] [pool.rs:146] ["scheduling task [split check worker] Split Check Task for 176, start_key: None, end_key: None, auto_split: true"] [thread_id=92] [2025/04/06 08:27:19.470 +08:00] [DEBUG] [pool.rs:146] ["scheduling task report min resolved ts: store 1, resolved ts 457152719441625089"] [thread_id=91] [2025/04/06 08:27:19.470 +08:00] [DEBUG] [split_check.rs:607] ["executing task"] [policy=Scan] [end_key=7A7480000000000000FFBD5F698000000000FF0000030380000000FF05F5E56500000000FC] [start_key=7A7480000000000000FFBD5F698000000000FF0000030380000000FF0000006500000000FB] [is_key_range=false] [region_id=178] [thread_id=16] [2025/04/06 08:27:19.470 +08:00] [DEBUG] [pd.rs:2055] ["executing task"] [task="report min resolved ts: store 1, resolved ts 457152719441625089"] [thread_id=30] [2025/04/06 08:27:19.470 +08:00] [DEBUG] [size.rs:196] ["approximate size less than threshold, does not need to do split check"] [threshold=402653184] [size=92758] [region_id=178] [thread_id=16] [2025/04/06 08:27:19.470 +08:00] [DEBUG] [util.rs:486] ["request sent: 1"] [thread_id=30] [2025/04/06 08:27:19.470 +08:00] [DEBUG] [keys.rs:186] ["approximate keys less than threshold, does not need to do split check"] [threshold=3840000] [keys=1302] [region_id=178] [thread_id=16] [2025/04/06 08:27:19.470 +08:00] [DEBUG] [split_check.rs:621] ["skip split check"] [end_key=7A7480000000000000FFBD5F698000000000FF0000030380000000FF05F5E56500000000FC] [start_key=7A7480000000000000FFBD5F698000000000FF0000030380000000FF0000006500000000FB] [is_key_range=false] [region_id=178] [thread_id=16] [2025/04/06 08:27:19.470 +08:00] [DEBUG] [split_check.rs:607] ["executing task"] [policy=Scan] [end_key=7A7480000000000000FFBD5F698000000000FF0000030380000000FF0000006500000000FB] [start_key=7A7480000000000000FFBD00000000000000F8] [is_key_range=false] [region_id=176] [thread_id=16] [2025/04/06 08:27:19.470 +08:00] [DEBUG] [size.rs:196] ["approximate size less than threshold, does not need to do split check"] [threshold=402653184] [size=192019] [region_id=176] [thread_id=16] [2025/04/06 08:27:19.470 +08:00] [DEBUG] [keys.rs:186] ["approximate keys less than threshold, does not need to do split check"] [threshold=3840000] [keys=1301] [region_id=176] [thread_id=16] [2025/04/06 08:27:19.471 +08:00] [DEBUG] [split_check.rs:621] ["skip split check"] [end_key=7A7480000000000000FFBD5F698000000000FF0000030380000000FF0000006500000000FB] [start_key=7A7480000000000000FFBD00000000000000F8] [is_key_range=false] [region_id=176] [thread_id=16] Create changefeed successfully! ID: b93c79fe-e6e3-491f-a0b9-723efd8ea267 Info: {"upstream_id":7489990138194206589,"namespace":"default","id":"b93c79fe-e6e3-491f-a0b9-723efd8ea267","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/?max-txn-row=1","create_time":"2025-04-06T08:27:19.483432904+08:00","start_ts":457152720916447233,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457152720916447233,"checkpoint_ts":457152720916447233,"checkpoint_time":"2025-04-06 08:27:14.453"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x + tso='457152721912332289 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 457152721912332289 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Sun Apr 6 08:27:19 CST 2025] <<<<<< START cdc server in generate_column case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.generate_column.3227632278.out server --log-file /tmp/tidb_cdc_test/generate_column/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/generate_column/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + set +x start tidb cluster in /tmp/tidb_cdc_test/sequence Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release check_redo_resolved_ts e4e316b2-866d-4639-852a-40150af2b5e8 457152721264050179 file:///tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/redo /tmp/tidb_cdc_test/consistent_replicate_storage_file_large_value/cdc_data/redo/e4e316b2-866d-4639-852a-40150af2b5e8/meta run task successfully wait process cdc.test exit for 1-th time... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 06 Apr 2025 00:27:22 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b15e6c1e-18b5-43f0-a235-a3be03f7883f {"id":"b15e6c1e-18b5-43f0-a235-a3be03f7883f","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899240} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087e7342e4 b15e6c1e-18b5-43f0-a235-a3be03f7883f /tidb/cdc/default/default/upstream/7489990153678648305 {"id":7489990153678648305,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b15e6c1e-18b5-43f0-a235-a3be03f7883f {"id":"b15e6c1e-18b5-43f0-a235-a3be03f7883f","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899240} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087e7342e4 b15e6c1e-18b5-43f0-a235-a3be03f7883f /tidb/cdc/default/default/upstream/7489990153678648305 {"id":7489990153678648305,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b15e6c1e-18b5-43f0-a235-a3be03f7883f {"id":"b15e6c1e-18b5-43f0-a235-a3be03f7883f","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899240} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087e7342e4 b15e6c1e-18b5-43f0-a235-a3be03f7883f /tidb/cdc/default/default/upstream/7489990153678648305 {"id":7489990153678648305,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.generate_column.cli.32332.out cli changefeed create --start-ts=457152721912332289 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ Create changefeed successfully! ID: 4165af80-f782-4886-9f4c-033d9aa85df7 Info: {"upstream_id":7489990153678648305,"namespace":"default","id":"4165af80-f782-4886-9f4c-033d9aa85df7","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-06T08:27:23.135380991+08:00","start_ts":457152721912332289,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457152721912332289,"checkpoint_ts":457152721912332289,"checkpoint_time":"2025-04-06 08:27:18.252"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... wait process cdc.test exit for 2-th time... Verifying downstream PD is started... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [2025/04/06 08:27:23.716 +08:00] [INFO] [helper.go:54] ["init log"] [file=] [level=info] [2025/04/06 08:27:23.717 +08:00] [INFO] [apply.go:78] ["set memory limit"] [memoryLimit=10737418240] [2025/04/06 08:27:23.717 +08:00] [INFO] [apply.go:94] ["Start http pprof server"] [addr=127.0.0.1:6060] [2025/04/06 08:27:23.717 +08:00] [INFO] [mem_quota.go:82] ["New memory quota"] [namespace=default] [changefeed=redo-applier] [total=1073741824] [2025/04/06 08:27:23.717 +08:00] [INFO] [redo.go:167] ["apply redo log starts"] [checkpointTs=457152715850514436] [resolvedTs=457152722089541633] [2025/04/06 08:27:23.718 +08:00] [INFO] [file.go:114] ["succeed to download and sort redo logs"] [type=ddl] [duration=1.029692ms] [2025/04/06 08:27:23.718 +08:00] [INFO] [tz.go:35] ["Use the timezone of the TiCDC server machine"] [timezoneName=System] [timezone=Asia/Shanghai] [2025/04/06 08:27:23.718 +08:00] [WARN] [config.go:428] ["Because time-zone is not specified, the timezone of the TiCDC server will be used. We recommend that you specify the time-zone explicitly. Please make sure that the timezone of the TiCDC server, sink-uri and the downstream database are consistent. If the downstream database does not load the timezone information, you can refer to https://dev.mysql.com/doc/refman/8.0/en/mysql-tzinfo-to-sql.html."] [timezone=Asia/Shanghai] [2025/04/06 08:27:23.718 +08:00] [ERROR] [config.go:216] ["The TiDB source ID should never be set to 0. Please report it as a bug. The default value will be used: 1."] [tidbSourceID=0] [2025/04/06 08:27:23.718 +08:00] [INFO] [file.go:114] ["succeed to download and sort redo logs"] [type=row] [duration=1.072146ms] [2025/04/06 08:27:23.724 +08:00] [INFO] [db_helper.go:161] ["sink uri is configured"] [dsn="normal:******@tcp(127.0.0.1:3306)/?interpolateParams=true&multiStatements=true&allow_auto_random_explicit_insert=1&charset=utf8mb4&foreign_key_checks=0&maxAllowedPacket=0&readTimeout=2m&sql_mode=%22ONLY_FULL_GROUP_BY%2CNO_AUTO_VALUE_ON_ZERO%2CIGNORE_SPACE%2CNO_ENGINE_SUBSTITUTION%2CALLOW_INVALID_DATES%22&tidb_enable_external_ts_read=%22OFF%22&tidb_placement_mode=%22ignore%22&tidb_txn_mode=optimistic&time_zone=%22Asia%2FShanghai%22&timeout=2m&transaction_isolation=%22READ-COMMITTED%22&writeTimeout=2m"] [2025/04/06 08:27:23.727 +08:00] [INFO] [mysql.go:195] ["MySQL backends is created"] [changefeed=.] [workerCount=16] [forceReplicate=false] [2025/04/06 08:27:23.729 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/06 08:27:23.729 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/06 08:27:23.729 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/06 08:27:23.729 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/06 08:27:23.729 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/06 08:27:23.729 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/06 08:27:23.729 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/06 08:27:23.729 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/06 08:27:23.729 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/06 08:27:23.729 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/06 08:27:23.729 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/06 08:27:23.729 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/06 08:27:23.729 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/06 08:27:23.729 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/06 08:27:23.729 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/06 08:27:23.729 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/06 08:27:23.729 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=1] [2025/04/06 08:27:23.729 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=4] [2025/04/06 08:27:23.729 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=2] [2025/04/06 08:27:23.729 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=7] [2025/04/06 08:27:23.729 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=0] [2025/04/06 08:27:23.729 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=6] [2025/04/06 08:27:23.729 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=9] [2025/04/06 08:27:23.729 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=12] [2025/04/06 08:27:23.729 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=10] [2025/04/06 08:27:23.729 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=11] [2025/04/06 08:27:23.729 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=14] [2025/04/06 08:27:23.729 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=15] [2025/04/06 08:27:23.729 +08:00] [INFO] [tz.go:35] ["Use the timezone of the TiCDC server machine"] [timezoneName=System] [timezone=Asia/Shanghai] [2025/04/06 08:27:23.729 +08:00] [WARN] [config.go:428] ["Because time-zone is not specified, the timezone of the TiCDC server will be used. We recommend that you specify the time-zone explicitly. Please make sure that the timezone of the TiCDC server, sink-uri and the downstream database are consistent. If the downstream database does not load the timezone information, you can refer to https://dev.mysql.com/doc/refman/8.0/en/mysql-tzinfo-to-sql.html."] [timezone=Asia/Shanghai] [2025/04/06 08:27:23.729 +08:00] [ERROR] [config.go:216] ["The TiDB source ID should never be set to 0. Please report it as a bug. The default value will be used: 1."] [tidbSourceID=0] [2025/04/06 08:27:23.729 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=13] [2025/04/06 08:27:23.729 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=8] [2025/04/06 08:27:23.729 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=3] [2025/04/06 08:27:23.729 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=5] [2025/04/06 08:27:23.735 +08:00] [INFO] [db_helper.go:161] ["sink uri is configured"] [dsn="normal:******@tcp(127.0.0.1:3306)/?interpolateParams=true&multiStatements=true&allow_auto_random_explicit_insert=1&charset=utf8mb4&foreign_key_checks=0&maxAllowedPacket=0&readTimeout=2m&sql_mode=%22NO_ENGINE_SUBSTITUTION%2CALLOW_INVALID_DATES%2CONLY_FULL_GROUP_BY%2CNO_AUTO_VALUE_ON_ZERO%2CIGNORE_SPACE%22&tidb_enable_external_ts_read=%22OFF%22&tidb_placement_mode=%22ignore%22&tidb_txn_mode=optimistic&time_zone=%22Asia%2FShanghai%22&timeout=2m&transaction_isolation=%22READ-COMMITTED%22&writeTimeout=2m"] [2025/04/06 08:27:23.737 +08:00] [WARN] [mysql_ddl_sink.go:291] ["please set `has-vector-type` to be true if a column is vector type when the downstream is not TiDB or TiDB version less than specify version"] [hasVectorType=false] [supportVectorVersion=8.4.0] [2025/04/06 08:27:23.737 +08:00] [INFO] [mysql_ddl_sink.go:122] ["MySQL DDL sink is created"] [namespace=] [changefeed=] [2025/04/06 08:27:23.737 +08:00] [ERROR] [redo.go:272] ["ignore unsupported DDL"] [ddl="{\"StartTs\":457152715837669410,\"CommitTs\":457152715850514436,\"Query\":\"ALTER TABLE `consistent_replicate_storage_file_large_value`.`t1` EXCHANGE PARTITION `p3` WITH TABLE `consistent_replicate_storage_file_large_value`.`t2`\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"Schema\":\"consistent_replicate_storage_file_large_value\",\"Table\":\"t1\",\"TableID\":118,\"IsPartition\":true},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":42,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false,\"BDRRole\":\"\",\"SQLMode\":0}"] [2025/04/06 08:27:23.939 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [span="{tableID:123, startKey:7480000000000000ff7b5f720000000000fa, endKey:7480000000000000ff7b5f730000000000fa}"] [checkpointTs=457152722089541633] [2025/04/06 08:27:23.939 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [span="{tableID:123, startKey:7480000000000000ff7b5f720000000000fa, endKey:7480000000000000ff7b5f730000000000fa}"] [checkpointTs=457152722089541633] [2025/04/06 08:27:24.240 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [span="{tableID:126, startKey:7480000000000000ff7e5f720000000000fa, endKey:7480000000000000ff7e5f730000000000fa}"] [checkpointTs=457152722089541633] [2025/04/06 08:27:24.240 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [span="{tableID:126, startKey:7480000000000000ff7e5f720000000000fa, endKey:7480000000000000ff7e5f730000000000fa}"] [checkpointTs=457152722089541633] [2025/04/06 08:27:24.340 +08:00] [INFO] [redo.go:223] ["apply redo log finishes"] [appliedLogCount=8] [appliedDDLCount=0] [currentCheckpoint=457152722089541633] [2025/04/06 08:27:24.340 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=4] [2025/04/06 08:27:24.340 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=15] [2025/04/06 08:27:24.340 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=3] [2025/04/06 08:27:24.340 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=10] [2025/04/06 08:27:24.340 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=9] [2025/04/06 08:27:24.340 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=11] [2025/04/06 08:27:24.340 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=1] [2025/04/06 08:27:24.340 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=6] [2025/04/06 08:27:24.340 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=8] [2025/04/06 08:27:24.340 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=12] [2025/04/06 08:27:24.340 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=7] [2025/04/06 08:27:24.340 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=14] [2025/04/06 08:27:24.340 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=2] [2025/04/06 08:27:24.340 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=0] [2025/04/06 08:27:24.340 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=13] [2025/04/06 08:27:24.340 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=5] Apply redo log successfully Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x check diff successfully [Sun Apr 6 08:27:24 CST 2025] <<<<<< run test case consistent_replicate_storage_file_large_value success! >>>>>> count(*) 0 Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table generate_column.t1 not exists for 1-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) The 1 times to try to start tidb cluster... table generate_column.t1 exists table generate_column.t2 not exists for 1-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489990205615218002 TiDB Cluster ID. tikv_gc_leader_uuid 65821faf8ac000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-8-31x9h, pid:41928, start at 2025-04-06 08:27:26.141208451 +0800 CST m=+2.115397954 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:29:26.148 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:27:26.123 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:17:26.123 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489990205615218002 TiDB Cluster ID. tikv_gc_leader_uuid 65821faf8ac000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-8-31x9h, pid:41928, start at 2025-04-06 08:27:26.141208451 +0800 CST m=+2.115397954 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:29:26.148 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:27:26.123 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:17:26.123 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489990196330186749 TiDB Cluster ID. tikv_gc_leader_uuid 65821fafddc000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-8-31x9h, pid:42001, start at 2025-04-06 08:27:26.497694976 +0800 CST m=+2.412971289 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:29:26.504 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:27:26.506 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:17:26.506 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/cli_tls_with_auth/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/cli_tls_with_auth/tiflash/log/error.log arg matches is ArgMatches { args: {"log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/cli_tls_with_auth/tiflash/log/proxy.log"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/cli_tls_with_auth/tiflash-proxy.toml"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/cli_tls_with_auth/tiflash/db/proxy"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) The 1 times to try to start tls tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/cli_tls_with_auth Starting TLS PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 table generate_column.t2 exists check diff failed 1-th time, retry later start tidb cluster in /tmp/tidb_cdc_test/multi_source Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... Starting TLS TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489990213534857646 TiDB Cluster ID. tikv_gc_leader_uuid 65821fb3aa80014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-p-5tjc6, pid:14586, start at 2025-04-06 08:27:30.38515251 +0800 CST m=+2.451971729 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:29:30.393 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:27:30.396 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:17:30.396 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489990213534857646 TiDB Cluster ID. tikv_gc_leader_uuid 65821fb3aa80014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-p-5tjc6, pid:14586, start at 2025-04-06 08:27:30.38515251 +0800 CST m=+2.451971729 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:29:30.393 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:27:30.396 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:17:30.396 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489990222173982413 TiDB Cluster ID. tikv_gc_leader_uuid 65821fb39180002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-p-5tjc6, pid:14661, start at 2025-04-06 08:27:30.247860711 +0800 CST m=+2.254590706 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:29:30.255 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:27:30.246 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:17:30.246 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/sequence/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/sequence/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/sequence/tiflash/log/proxy.log"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/sequence/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/sequence/tiflash/db/proxy"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } check diff successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Sun Apr 6 08:27:33 CST 2025] <<<<<< run test case generate_column success! >>>>>> Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting TLS TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying TLS TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Sun Apr 6 08:27:34 CST 2025] <<<<<< START cdc server in sequence case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sequence.1593615938.out server --log-file /tmp/tidb_cdc_test/sequence/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/sequence/cdc_data --cluster-id default + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/consistent_replicate_storage_s3/run.sh using Sink-Type: mysql... <<================= * About to connect() to 127.0.0.1 port 24927 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:24927; Connection refused * Closing connection 0 You are running an older version of MinIO released 4 years ago Update: Run `mc admin update` Attempting encryption of all config, IAM users and policies on MinIO backend Endpoint: http://127.0.0.1:24927 Object API (Amazon S3 compatible): Go: https://docs.min.io/docs/golang-client-quickstart-guide Java: https://docs.min.io/docs/java-client-quickstart-guide Python: https://docs.min.io/docs/python-client-quickstart-guide JavaScript: https://docs.min.io/docs/javascript-client-quickstart-guide .NET: https://docs.min.io/docs/dotnet-client-quickstart-guide Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489990252901441927 TiDB Cluster ID. tikv_gc_leader_uuid 65821fb84740007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-8-31x9h, pid:43717, start at 2025-04-06 08:27:35.080329654 +0800 CST m=+1.319112338 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:29:35.086 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:27:35.069 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:17:35.069 +0800 All versions after safe point can be accessed. (DO NOT EDIT) + pd_host=127.0.0.1 + pd_port=2579 + is_tls=true + '[' true == true ']' ++ run_cdc_cli tso query --pd=https://127.0.0.1:2579 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_tls_with_auth.cli.43773.out cli tso query --pd=https://127.0.0.1:2579 * About to connect() to 127.0.0.1 port 24927 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 24927 (#0) > GET / HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:24927 > Accept: */* > < HTTP/1.1 403 Forbidden < Accept-Ranges: bytes < Content-Length: 226 < Content-Security-Policy: block-all-mixed-content < Content-Type: application/xml < Server: MinIO/RELEASE.2020-07-27T18-37-02Z < Vary: Origin < X-Amz-Request-Id: 183393225F4F7447 < X-Xss-Protection: 1; mode=block < Date: Sun, 06 Apr 2025 00:27:36 GMT < { [data not shown] * Connection #0 to host 127.0.0.1 left intact Bucket 's3://logbucket/' created The 1 times to try to start tidb cluster... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x + tso='457152726622011393 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 457152726622011393 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Sun Apr 6 08:27:37 CST 2025] <<<<<< START cdc server in cli_tls_with_auth case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates ']' + curl_status_cmd='curl --cacert /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client.pem --key /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client-key.pem --user ticdc:ticdc_secret -vsL --max-time 20 https://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_tls_with_auth.4382643828.out server --log-file /tmp/tidb_cdc_test/cli_tls_with_auth/cdc_cli_tls_with_auth_tls1.log --log-level debug --data-dir /tmp/tidb_cdc_test/cli_tls_with_auth/cdc_data_cli_tls_with_auth_tls1 --cluster-id default --config /tmp/tidb_cdc_test/cli_tls_with_auth/server.toml --ca /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/server.pem --key /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/server-key.pem --cert-allowed-cn client --addr 127.0.0.1:8300 --pd https://127.0.0.1:2579 + (( i = 0 )) + (( i <= 50 )) ++ curl --cacert /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client.pem --key /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client-key.pem --user ticdc:ticdc_secret -vsL --max-time 20 https://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 06 Apr 2025 00:27:37 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/871b8505-136b-4122-87e3-9bdd0d2f2302 {"id":"871b8505-136b-4122-87e3-9bdd0d2f2302","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899254} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087ead3cdc 871b8505-136b-4122-87e3-9bdd0d2f2302 /tidb/cdc/default/default/upstream/7489990213534857646 {"id":7489990213534857646,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/871b8505-136b-4122-87e3-9bdd0d2f2302 {"id":"871b8505-136b-4122-87e3-9bdd0d2f2302","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899254} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087ead3cdc 871b8505-136b-4122-87e3-9bdd0d2f2302 /tidb/cdc/default/default/upstream/7489990213534857646 {"id":7489990213534857646,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/871b8505-136b-4122-87e3-9bdd0d2f2302 {"id":"871b8505-136b-4122-87e3-9bdd0d2f2302","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899254} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087ead3cdc 871b8505-136b-4122-87e3-9bdd0d2f2302 /tidb/cdc/default/default/upstream/7489990213534857646 {"id":7489990213534857646,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sequence.cli.15998.out cli changefeed create --start-ts=457152726144647171 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ --config /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/sequence/conf/force_replicate.toml Create changefeed successfully! ID: 45e217ed-4a5b-491a-b67d-923be5c2cfbc Info: {"upstream_id":7489990213534857646,"namespace":"default","id":"45e217ed-4a5b-491a-b67d-923be5c2cfbc","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-06T08:27:37.88926333+08:00","start_ts":457152726144647171,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":true,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457152726144647171,"checkpoint_ts":457152726144647171,"checkpoint_time":"2025-04-06 08:27:34.397"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x table sequence_test.t1 not exists for 1-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489990257990128578 TiDB Cluster ID. tikv_gc_leader_uuid 65821fbb8ac0002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-v-hm9ds, pid:26679, start at 2025-04-06 08:27:38.427769995 +0800 CST m=+2.606668412 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:29:38.436 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:27:38.411 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:17:38.411 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489990257990128578 TiDB Cluster ID. tikv_gc_leader_uuid 65821fbb8ac0002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-v-hm9ds, pid:26679, start at 2025-04-06 08:27:38.427769995 +0800 CST m=+2.606668412 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:29:38.436 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:27:38.411 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:17:38.411 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489990257660947157 TiDB Cluster ID. tikv_gc_leader_uuid 65821fbba740008 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-v-hm9ds, pid:26751, start at 2025-04-06 08:27:38.540186313 +0800 CST m=+2.658903660 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:29:38.550 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:27:38.525 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:17:38.525 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/multi_source/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/multi_source/tiflash/log/error.log arg matches is ArgMatches { args: {"pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/multi_source/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/multi_source/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/multi_source/tiflash-proxy.toml"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } start tidb cluster in /tmp/tidb_cdc_test/consistent_replicate_storage_s3 Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... + (( i++ )) + (( i <= 50 )) ++ curl --cacert /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client.pem --key /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client-key.pem --user ticdc:ticdc_secret -vsL --max-time 20 https://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Initializing NSS with certpath: sql:/etc/pki/nssdb * CAfile: /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem CApath: none * NSS: client certificate from file * subject: CN=client * start date: Feb 18 07:48:00 2020 GMT * expire date: Jan 25 07:48:00 2120 GMT * common name: client * issuer: CN=My own CA,O=PingCAP,L=Beijing,ST=Beijing,C=CN * SSL connection using TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 * Server certificate: * subject: CN=tidb-server * start date: Feb 18 09:11:00 2020 GMT * expire date: Jan 25 09:11:00 2120 GMT * common name: tidb-server * issuer: CN=My own CA,O=PingCAP,L=Beijing,ST=Beijing,C=CN * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 06 Apr 2025 00:27:40 GMT < Content-Length: 1253 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6587fa2b-486c-41d3-ba54-aa6399c3dc32 {"id":"6587fa2b-486c-41d3-ba54-aa6399c3dc32","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899258} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/1f5d96087ecf4af0 6587fa2b-486c-41d3-ba54-aa6399c3dc32 /tidb/cdc/default/default/upstream/7489990252901441927 {"id":7489990252901441927,"pd-endpoints":"https://127.0.0.1:2579,https://127.0.0.1:2579","key-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/server-key.pem","cert-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/server.pem","ca-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem","cert-allowed-cn":["client","tidb-server"]}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6587fa2b-486c-41d3-ba54-aa6399c3dc32 {"id":"6587fa2b-486c-41d3-ba54-aa6399c3dc32","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899258} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/1f5d96087ecf4af0 6587fa2b-486c-41d3-ba54-aa6399c3dc32 /tidb/cdc/default/default/upstream/7489990252901441927 {"id":7489990252901441927,"pd-endpoints":"https://127.0.0.1:2579,https://127.0.0.1:2579","key-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/server-key.pem","cert-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/server.pem","ca-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem","cert-allowed-cn":["client","tidb-server"]}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6587fa2b-486c-41d3-ba54-aa6399c3dc32 {"id":"6587fa2b-486c-41d3-ba54-aa6399c3dc32","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899258} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/1f5d96087ecf4af0 6587fa2b-486c-41d3-ba54-aa6399c3dc32 /tidb/cdc/default/default/upstream/7489990252901441927 {"id":7489990252901441927,"pd-endpoints":"https://127.0.0.1:2579,https://127.0.0.1:2579","key-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/server-key.pem","cert-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/server.pem","ca-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem","cert-allowed-cn":["client","tidb-server"]}' + grep -q 'etcd info' + break + set +x table sequence_test.t1 exists check table exists success check diff failed 1-th time, retry later + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_source.cli.28000.out cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_tls_with_auth.cli.43903.out cli changefeed create --start-ts=457152726622011393 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ --tz=Asia/Shanghai -c=custom-changefeed-name [WARN] --tz is deprecated in changefeed settings. Create changefeed successfully! ID: custom-changefeed-name Info: {"upstream_id":7489990252901441927,"namespace":"default","id":"custom-changefeed-name","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-06T08:27:43.432884328+08:00","start_ts":457152726622011393,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457152726622011393,"checkpoint_ts":457152726622011393,"checkpoint_time":"2025-04-06 08:27:36.218"} PASS Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release coverage: 2.4% of statements in github.com/pingcap/tiflow/... check diff successfully ERROR 1146 (42S02) at line 1: Table 'sequence_test.seq0' doesn't exist table sequence_test.mark_table not exists for 1-th check, retry later \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } [Pipeline] // timeout [Pipeline] } + set +x + tso='457152728297897985 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 457152728297897985 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Sun Apr 6 08:27:44 CST 2025] <<<<<< START cdc server in multi_source case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_source.2803928041.out server --log-file /tmp/tidb_cdc_test/multi_source/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_source/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Cache not saved (ws/jenkins-pingcap-tiflow-periodics_cdc_integration_mysql_test-234/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 723 100 723 0 0 40213 0 --:--:-- --:--:-- --:--:-- 42529 + synced_status='{"synced":false,"sink_checkpoint_ts":"2025-04-06 08:25:20.918","puller_resolved_ts":"2025-04-06 08:25:20.918","last_synced_ts":"2025-04-06 08:25:15.618","now_ts":"2025-04-06 08:27:32.000","info":"Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' \u003e '\''Resolved-Ts'\'' \u003e '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait"}' ++ echo '{"synced":false,"sink_checkpoint_ts":"2025-04-06' '08:25:20.918","puller_resolved_ts":"2025-04-06' '08:25:20.918","last_synced_ts":"2025-04-06' '08:25:15.618","now_ts":"2025-04-06' '08:27:32.000","info":"Please' check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view ''\''TiKV-Details'\''' '\u003e' ''\''Resolved-Ts'\''' '\u003e' ''\''Max' Leader Resolved TS 'gap'\''' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please 'wait"}' ++ jq .synced + status=false + '[' false '!=' false ']' ++ echo '{"synced":false,"sink_checkpoint_ts":"2025-04-06' '08:25:20.918","puller_resolved_ts":"2025-04-06' '08:25:20.918","last_synced_ts":"2025-04-06' '08:25:15.618","now_ts":"2025-04-06' '08:27:32.000","info":"Please' check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view ''\''TiKV-Details'\''' '\u003e' ''\''Resolved-Ts'\''' '\u003e' ''\''Max' Leader Resolved TS 'gap'\''' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please 'wait"}' ++ jq -r .info + info='Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' + target_message='Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' + '[' 'Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' '!=' 'Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' ']' + cleanup_process cdc.test wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit + stop_tidb_cluster + run_case_with_unavailable_tidb conf/changefeed-redo.toml + rm -rf /tmp/tidb_cdc_test/synced_status_with_redo + mkdir -p /tmp/tidb_cdc_test/synced_status_with_redo + start_tidb_cluster --workdir /tmp/tidb_cdc_test/synced_status_with_redo shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory The 1 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } + set +x table test.simple not exists for 1-th check, retry later Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/synced_status_with_redo Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... table sequence_test.mark_table exists wait process cdc.test exit for 1-th time... table test.simple exists table test.`simple-dash` not exists for 1-th check, retry later wait process cdc.test exit for 2-th time... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 06 Apr 2025 00:27:47 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0817bcf6-4b81-45dd-a176-e2dc158a28a0 {"id":"0817bcf6-4b81-45dd-a176-e2dc158a28a0","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899264} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087ed204e6 0817bcf6-4b81-45dd-a176-e2dc158a28a0 /tidb/cdc/default/default/upstream/7489990257990128578 {"id":7489990257990128578,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0817bcf6-4b81-45dd-a176-e2dc158a28a0 {"id":"0817bcf6-4b81-45dd-a176-e2dc158a28a0","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899264} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087ed204e6 0817bcf6-4b81-45dd-a176-e2dc158a28a0 /tidb/cdc/default/default/upstream/7489990257990128578 {"id":7489990257990128578,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0817bcf6-4b81-45dd-a176-e2dc158a28a0 {"id":"0817bcf6-4b81-45dd-a176-e2dc158a28a0","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899264} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087ed204e6 0817bcf6-4b81-45dd-a176-e2dc158a28a0 /tidb/cdc/default/default/upstream/7489990257990128578 {"id":7489990257990128578,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_source.cli.28093.out cli changefeed create --start-ts=457152728297897985 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Sun Apr 6 08:27:47 CST 2025] <<<<<< run test case sequence success! >>>>>> Create changefeed successfully! ID: adfa97ec-987b-4ec1-8010-8aa9d0d41b3b Info: {"upstream_id":7489990257990128578,"namespace":"default","id":"adfa97ec-987b-4ec1-8010-8aa9d0d41b3b","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-06T08:27:47.624269227+08:00","start_ts":457152728297897985,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457152728297897985,"checkpoint_ts":457152728297897985,"checkpoint_time":"2025-04-06 08:27:42.611"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + set +x VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489990294254066684 TiDB Cluster ID. tikv_gc_leader_uuid 65821fc4458000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-6-zhp59, pid:15896, start at 2025-04-06 08:27:47.368882383 +0800 CST m=+2.359518782 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:29:47.377 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:27:47.350 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:17:47.350 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489990294254066684 TiDB Cluster ID. tikv_gc_leader_uuid 65821fc4458000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-6-zhp59, pid:15896, start at 2025-04-06 08:27:47.368882383 +0800 CST m=+2.359518782 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:29:47.377 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:27:47.350 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:17:47.350 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489990296634592521 TiDB Cluster ID. tikv_gc_leader_uuid 65821fc45dc0003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-6-zhp59, pid:15964, start at 2025-04-06 08:27:47.451025399 +0800 CST m=+2.377599207 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:29:47.460 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:27:47.447 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:17:47.447 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/consistent_replicate_storage_s3/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/consistent_replicate_storage_s3/tiflash/log/error.log arg matches is ArgMatches { args: {"addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/consistent_replicate_storage_s3/tiflash-proxy.toml"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/consistent_replicate_storage_s3/tiflash/db/proxy"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/consistent_replicate_storage_s3/tiflash/log/proxy.log"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table test.`simple-dash` exists + endpoints=https://127.0.0.1:2579 + changefeed_id=custom-changefeed-name + expected_state=normal + error_msg=null + tls_dir=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates + [[ https://127.0.0.1:2579 =~ https ]] ++ cdc cli changefeed query --ca=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client.pem --key=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client-key.pem --pd=https://127.0.0.1:2579 -c custom-changefeed-name -s + info='{ "upstream_id": 7489990252901441927, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 457152727015227401, "checkpoint_time": "2025-04-06 08:27:37.718", "error": null }' + echo '{ "upstream_id": 7489990252901441927, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 457152727015227401, "checkpoint_time": "2025-04-06 08:27:37.718", "error": null }' { "upstream_id": 7489990252901441927, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 457152727015227401, "checkpoint_time": "2025-04-06 08:27:37.718", "error": null } ++ echo '{' '"upstream_id":' 7489990252901441927, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 457152727015227401, '"checkpoint_time":' '"2025-04-06' '08:27:37.718",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7489990252901441927, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 457152727015227401, '"checkpoint_time":' '"2025-04-06' '08:27:37.718",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] changefeed count 1 check pass, pd_addr: https://127.0.0.1:2579 Error: [CDC:ErrChangefeedUpdateRefused]changefeed update error: can only update changefeed config when it is stopped or failed update changefeed config should fail when changefeed is running, got Diff of changefeed config: {Type:update Path:[Config CaseSensitive] From:false To:true} {Type:update Path:[Config Scheduler EnableTableAcrossNodes] From:false To:true} + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_tls_with_auth.cli.44102.out cli changefeed --changefeed-id custom-changefeed-name pause PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Sun Apr 6 08:27:51 CST 2025] <<<<<< START cdc server in consistent_replicate_storage_s3 case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.consistent_replicate_storage_s3.1729217294.out server --log-file /tmp/tidb_cdc_test/consistent_replicate_storage_s3/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/consistent_replicate_storage_s3/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + set +x table changefeed_dup_error_restart.finish_mark_1 exists check diff successfully ***************** properties ***************** "updateproportion"="0" "scanproportion"="0" "mysql.host"="127.0.0.1" "mysql.port"="4000" "mysql.db"="changefeed_dup_error_restart" "readproportion"="0" "operationcount"="0" "dotransactions"="false" "requestdistribution"="uniform" "mysql.user"="root" "insertproportion"="0" "recordcount"="10" "readallfields"="true" "threadcount"="2" "workload"="core" ********************************************** Run finished, takes 5.043743ms INSERT - Takes(s): 0.0, Count: 10, OPS: 2628.8, Avg(us): 843, Min(us): 509, Max(us): 2239, 95th(us): 3000, 99th(us): 3000 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 06 Apr 2025 00:27:54 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1fd31338-2c9e-4421-821c-161e6820e930 {"id":"1fd31338-2c9e-4421-821c-161e6820e930","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899271} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087ef5ead7 1fd31338-2c9e-4421-821c-161e6820e930 /tidb/cdc/default/default/upstream/7489990294254066684 {"id":7489990294254066684,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1fd31338-2c9e-4421-821c-161e6820e930 {"id":"1fd31338-2c9e-4421-821c-161e6820e930","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899271} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087ef5ead7 1fd31338-2c9e-4421-821c-161e6820e930 /tidb/cdc/default/default/upstream/7489990294254066684 {"id":7489990294254066684,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1fd31338-2c9e-4421-821c-161e6820e930 {"id":"1fd31338-2c9e-4421-821c-161e6820e930","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899271} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087ef5ead7 1fd31338-2c9e-4421-821c-161e6820e930 /tidb/cdc/default/default/upstream/7489990294254066684 {"id":7489990294254066684,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489990317567157426 TiDB Cluster ID. tikv_gc_leader_uuid 65821fc9f74003e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-d-7dnbd, pid:15297, start at 2025-04-06 08:27:53.209333773 +0800 CST m=+2.300628324 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:29:53.262 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:27:53.257 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:17:53.257 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489990317567157426 TiDB Cluster ID. tikv_gc_leader_uuid 65821fc9f74003e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-d-7dnbd, pid:15297, start at 2025-04-06 08:27:53.209333773 +0800 CST m=+2.300628324 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:29:53.262 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:27:53.257 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:17:53.257 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489990320372827555 TiDB Cluster ID. tikv_gc_leader_uuid 65821fca0a4000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-d-7dnbd, pid:15375, start at 2025-04-06 08:27:53.274889394 +0800 CST m=+2.302202714 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:29:53.285 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:27:53.279 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:17:53.279 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... + endpoints=https://127.0.0.1:2579 + changefeed_id=custom-changefeed-name + expected_state=stopped + error_msg=null + tls_dir=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates + [[ https://127.0.0.1:2579 =~ https ]] ++ cdc cli changefeed query --ca=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client.pem --key=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client-key.pem --pd=https://127.0.0.1:2579 -c custom-changefeed-name -s + info='{ "upstream_id": 7489990252901441927, "namespace": "default", "id": "custom-changefeed-name", "state": "stopped", "checkpoint_tso": 457152730187169797, "checkpoint_time": "2025-04-06 08:27:49.818", "error": null }' + echo '{ "upstream_id": 7489990252901441927, "namespace": "default", "id": "custom-changefeed-name", "state": "stopped", "checkpoint_tso": 457152730187169797, "checkpoint_time": "2025-04-06 08:27:49.818", "error": null }' { "upstream_id": 7489990252901441927, "namespace": "default", "id": "custom-changefeed-name", "state": "stopped", "checkpoint_tso": 457152730187169797, "checkpoint_time": "2025-04-06 08:27:49.818", "error": null } ++ echo '{' '"upstream_id":' 7489990252901441927, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"stopped",' '"checkpoint_tso":' 457152730187169797, '"checkpoint_time":' '"2025-04-06' '08:27:49.818",' '"error":' null '}' ++ jq -r .state + state=stopped + [[ ! stopped == \s\t\o\p\p\e\d ]] ++ echo '{' '"upstream_id":' 7489990252901441927, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"stopped",' '"checkpoint_tso":' 457152730187169797, '"checkpoint_time":' '"2025-04-06' '08:27:49.818",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_tls_with_auth.cli.44191.out cli changefeed update --pd=https://127.0.0.1:2579 --config=/tmp/tidb_cdc_test/cli_tls_with_auth/changefeed.toml --no-confirm --changefeed-id custom-changefeed-name TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/synced_status_with_redo/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/synced_status_with_redo/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/synced_status_with_redo/tiflash/log/proxy.log"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/synced_status_with_redo/tiflash/db/proxy"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/synced_status_with_redo/tiflash-proxy.toml"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [2025/04/06 08:27:55.016 +08:00] [INFO] [main.go:86] ["running ddl test: 0 createDropSchemaDDL"] [2025/04/06 08:27:55.278 +08:00] [INFO] [main.go:221] ["1 insert success: 100"] [2025/04/06 08:27:55.278 +08:00] [INFO] [main.go:221] ["0 insert success: 100"] [2025/04/06 08:27:55.507 +08:00] [INFO] [main.go:221] ["0 insert success: 200"] [2025/04/06 08:27:55.509 +08:00] [INFO] [main.go:235] ["0 delete success: 100"] [2025/04/06 08:27:55.521 +08:00] [INFO] [main.go:221] ["1 insert success: 200"] [2025/04/06 08:27:55.523 +08:00] [INFO] [main.go:235] ["1 delete success: 100"] Diff of changefeed config: {Type:update Path:[Config CaseSensitive] From:false To:true} {Type:update Path:[Config Scheduler EnableTableAcrossNodes] From:false To:true} Update changefeed config successfully! ID: custom-changefeed-name Info: {"upstream_id":7489990252901441927,"namespace":"default","id":"custom-changefeed-name","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-06T08:27:43.432884328+08:00","start_ts":457152726622011393,"admin_job_type":1,"config":{"memory_quota":1073741824,"case_sensitive":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"delete_only_output_handle_key_columns":null,"content_compatible":null,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"stopped","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":0,"checkpoint_ts":457152730187169797,"checkpoint_time":"2025-04-06 08:27:49.818"} PASS coverage: 2.9% of statements in github.com/pingcap/tiflow/... ***************** properties ***************** "scanproportion"="0" "readproportion"="0" "dotransactions"="false" "mysql.db"="consistent_replicate_storage_s3" "mysql.user"="root" "updateproportion"="0" "workload"="core" "fieldcount"="100" "requestdistribution"="uniform" "recordcount"="5000" "readallfields"="true" "operationcount"="0" "mysql.host"="127.0.0.1" "mysql.port"="4000" "threadcount"="10" "insertproportion"="0" ********************************************** [2025/04/06 08:27:56.761 +08:00] [INFO] [main.go:221] ["0 insert success: 300"] [2025/04/06 08:27:56.772 +08:00] [INFO] [main.go:221] ["1 insert success: 300"] [2025/04/06 08:27:56.984 +08:00] [INFO] [main.go:221] ["0 insert success: 400"] [2025/04/06 08:27:56.988 +08:00] [INFO] [main.go:235] ["0 delete success: 200"] [2025/04/06 08:27:56.999 +08:00] [INFO] [main.go:221] ["1 insert success: 400"] [2025/04/06 08:27:57.005 +08:00] [INFO] [main.go:235] ["1 delete success: 200"] [2025/04/06 08:27:57.202 +08:00] [INFO] [main.go:221] ["0 insert success: 500"] [2025/04/06 08:27:57.220 +08:00] [INFO] [main.go:221] ["1 insert success: 500"] + cd /tmp/tidb_cdc_test/synced_status_with_redo ++ run_cdc_cli_tso_query 127.0.0.1 2379 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status_with_redo.cli.16699.out cli tso query --pd=http://127.0.0.1:2379 + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_tls_with_auth.cli.44235.out cli changefeed --changefeed-id custom-changefeed-name resume [2025/04/06 08:27:52.445 +08:00] [WARN] [diff.go:766] ["rows is not equal"] [table=`test`.`ntest`] [where="((TRUE) AND TRUE)"] [cost=1m31.863396165s] [2025/04/06 08:27:52.448 +08:00] [WARN] [diff.go:384] ["check chunk data not equal"] [chunk="{\"id\":0,\"bounds\":[],\"where\":\"((TRUE) AND TRUE)\",\"args\":null,\"state\":\"failed\"}"] [2025/04/06 08:27:57.996 +08:00] [INFO] [dailytest.go:68] ["test pass!!!"] PASS coverage: 2.1% of statements in github.com/pingcap/tiflow/... Run finished, takes 2.285657299s INSERT - Takes(s): 2.3, Count: 5000, OPS: 2197.0, Avg(us): 4219, Min(us): 1162, Max(us): 67551, 95th(us): 8000, 99th(us): 59000 [2025/04/06 08:27:58.440 +08:00] [INFO] [main.go:221] ["0 insert success: 600"] [2025/04/06 08:27:58.444 +08:00] [INFO] [main.go:235] ["0 delete success: 300"] [2025/04/06 08:27:58.460 +08:00] [INFO] [main.go:221] ["1 insert success: 600"] [2025/04/06 08:27:58.467 +08:00] [INFO] [main.go:235] ["1 delete success: 300"] [2025/04/06 08:27:58.681 +08:00] [INFO] [main.go:221] ["0 insert success: 700"] [2025/04/06 08:27:58.691 +08:00] [INFO] [main.go:221] ["1 insert success: 700"] wait process cdc.test exit for 1-th time... + set +x + tso='457152732222193665 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 457152732222193665 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + start_ts=457152732222193665 + run_cdc_server --workdir /tmp/tidb_cdc_test/synced_status_with_redo --binary cdc.test [Sun Apr 6 08:27:59 CST 2025] <<<<<< START cdc server in synced_status_with_redo case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status_with_redo.1674016742.out server --log-file /tmp/tidb_cdc_test/synced_status_with_redo/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/synced_status_with_redo/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 wait process cdc.test exit for 2-th time... + set +x \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Sun Apr 6 08:27:59 CST 2025] <<<<<< run test case cdc success! >>>>>> [Pipeline] // timeout [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-periodics_cdc_integration_mysql_test-234/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [2025/04/06 08:27:59.914 +08:00] [INFO] [main.go:221] ["0 insert success: 800"] [2025/04/06 08:27:59.920 +08:00] [INFO] [main.go:235] ["0 delete success: 400"] [2025/04/06 08:27:59.923 +08:00] [INFO] [main.go:221] ["1 insert success: 800"] [2025/04/06 08:27:59.934 +08:00] [INFO] [main.go:235] ["1 delete success: 400"] [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } table consistent_replicate_storage_s3.usertable exists table consistent_replicate_storage_s3.check1 not exists for 1-th check, retry later [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } table consistent_replicate_storage_s3.check1 not exists for 2-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 06 Apr 2025 00:28:02 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/371ab12c-aa13-451a-bd53-72c259ab2d8a {"id":"371ab12c-aa13-451a-bd53-72c259ab2d8a","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899279} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087f08faec 371ab12c-aa13-451a-bd53-72c259ab2d8a /tidb/cdc/default/default/upstream/7489990317567157426 {"id":7489990317567157426,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/371ab12c-aa13-451a-bd53-72c259ab2d8a {"id":"371ab12c-aa13-451a-bd53-72c259ab2d8a","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899279} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087f08faec 371ab12c-aa13-451a-bd53-72c259ab2d8a /tidb/cdc/default/default/upstream/7489990317567157426 {"id":7489990317567157426,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/371ab12c-aa13-451a-bd53-72c259ab2d8a {"id":"371ab12c-aa13-451a-bd53-72c259ab2d8a","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899279} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087f08faec 371ab12c-aa13-451a-bd53-72c259ab2d8a /tidb/cdc/default/default/upstream/7489990317567157426 {"id":7489990317567157426,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + config_path=conf/changefeed-redo.toml + SINK_URI='mysql://root@127.0.0.1:3306/?max-txn-row=1' + run_cdc_cli changefeed create --start-ts=457152732222193665 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/synced_status_with_redo/conf/changefeed-redo.toml + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status_with_redo.cli.16797.out cli changefeed create --start-ts=457152732222193665 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/synced_status_with_redo/conf/changefeed-redo.toml Create changefeed successfully! ID: test-1 Info: {"upstream_id":7489990317567157426,"namespace":"default","id":"test-1","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2025-04-06T08:28:02.474204899+08:00","start_ts":457152732222193665,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"eventual","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"storage":"file:///tmp/tidb_cdc_test/synced_status/redo","use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":120,"checkpoint_interval":20}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457152732222193665,"checkpoint_ts":457152732222193665,"checkpoint_time":"2025-04-06 08:27:57.581"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... + endpoints=https://127.0.0.1:2579 + changefeed_id=custom-changefeed-name + expected_state=normal + error_msg=null + tls_dir=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates + [[ https://127.0.0.1:2579 =~ https ]] ++ cdc cli changefeed query --ca=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client.pem --key=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client-key.pem --pd=https://127.0.0.1:2579 -c custom-changefeed-name -s + info='{ "upstream_id": 7489990252901441927, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 457152733333159944, "checkpoint_time": "2025-04-06 08:28:01.819", "error": null }' + echo '{ "upstream_id": 7489990252901441927, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 457152733333159944, "checkpoint_time": "2025-04-06 08:28:01.819", "error": null }' { "upstream_id": 7489990252901441927, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 457152733333159944, "checkpoint_time": "2025-04-06 08:28:01.819", "error": null } ++ echo '{' '"upstream_id":' 7489990252901441927, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 457152733333159944, '"checkpoint_time":' '"2025-04-06' '08:28:01.819",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7489990252901441927, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 457152733333159944, '"checkpoint_time":' '"2025-04-06' '08:28:01.819",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_tls_with_auth.cli.44327.out cli changefeed --changefeed-id custom-changefeed-name remove Changefeed remove successfully. ID: custom-changefeed-name CheckpointTs: 457152733333159944 SinkURI: mysql://normal:xxxxx@127.0.0.1:3306/ PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... + set +x + run_sql 'USE TEST;Create table t1(a int primary key, b int);insert into t1 values(1,2);insert into t1 values(2,3);' + check_table_exists test.t1 127.0.0.1 3306 table test.t1 exists + sleep 5 + set +x table consistent_replicate_storage_s3.check1 not exists for 3-th check, retry later table consistent_replicate_storage_s3.check1 not exists for 4-th check, retry later [2025/04/06 08:28:05.528 +08:00] [INFO] [main.go:86] ["running ddl test: 1 truncateDDL"] [2025/04/06 08:28:05.783 +08:00] [INFO] [main.go:221] ["0 insert success: 100"] [2025/04/06 08:28:05.801 +08:00] [INFO] [main.go:221] ["1 insert success: 100"] [2025/04/06 08:28:06.004 +08:00] [INFO] [main.go:235] ["0 delete success: 100"] [2025/04/06 08:28:06.005 +08:00] [INFO] [main.go:221] ["0 insert success: 200"] [2025/04/06 08:28:06.027 +08:00] [INFO] [main.go:221] ["1 insert success: 200"] [2025/04/06 08:28:06.029 +08:00] [INFO] [main.go:235] ["1 delete success: 100"] [2025/04/06 08:28:06.212 +08:00] [INFO] [main.go:221] ["0 insert success: 300"] [2025/04/06 08:28:06.252 +08:00] [INFO] [main.go:221] ["1 insert success: 300"] [2025/04/06 08:28:06.434 +08:00] [INFO] [main.go:235] ["0 delete success: 200"] [2025/04/06 08:28:06.435 +08:00] [INFO] [main.go:221] ["0 insert success: 400"] [2025/04/06 08:28:06.487 +08:00] [INFO] [main.go:221] ["1 insert success: 400"] [2025/04/06 08:28:06.495 +08:00] [INFO] [main.go:235] ["1 delete success: 200"] [2025/04/06 08:28:06.655 +08:00] [INFO] [main.go:221] ["0 insert success: 500"] [2025/04/06 08:28:06.747 +08:00] [INFO] [main.go:221] ["1 insert success: 500"] [2025/04/06 08:28:06.906 +08:00] [INFO] [main.go:235] ["0 delete success: 300"] [2025/04/06 08:28:06.907 +08:00] [INFO] [main.go:221] ["0 insert success: 600"] [2025/04/06 08:28:06.989 +08:00] [INFO] [main.go:221] ["1 insert success: 600"] [2025/04/06 08:28:07.001 +08:00] [INFO] [main.go:235] ["1 delete success: 300"] [2025/04/06 08:28:07.136 +08:00] [INFO] [main.go:221] ["0 insert success: 700"] [2025/04/06 08:28:07.225 +08:00] [INFO] [main.go:221] ["1 insert success: 700"] [2025/04/06 08:28:07.366 +08:00] [INFO] [main.go:235] ["0 delete success: 400"] [2025/04/06 08:28:07.367 +08:00] [INFO] [main.go:221] ["0 insert success: 800"] [2025/04/06 08:28:07.468 +08:00] [INFO] [main.go:221] ["1 insert success: 800"] [2025/04/06 08:28:07.485 +08:00] [INFO] [main.go:235] ["1 delete success: 400"] [2025/04/06 08:28:07.600 +08:00] [INFO] [main.go:221] ["0 insert success: 900"] [2025/04/06 08:28:07.701 +08:00] [INFO] [main.go:221] ["1 insert success: 900"] [2025/04/06 08:28:07.833 +08:00] [INFO] [main.go:235] ["0 delete success: 500"] [2025/04/06 08:28:07.834 +08:00] [INFO] [main.go:221] ["0 insert success: 1000"] [2025/04/06 08:28:07.931 +08:00] [INFO] [main.go:221] ["1 insert success: 1000"] [2025/04/06 08:28:07.954 +08:00] [INFO] [main.go:235] ["1 delete success: 500"] changefeed count 0 check pass, pd_addr: https://127.0.0.1:2579 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_tls_with_auth.cli.44401.out cli changefeed create --sink-uri=mysql://normal:123456@127.0.0.1:3306/ --tz=Asia/Shanghai -c=custom-changefeed-name [2025/04/06 08:28:08.071 +08:00] [INFO] [main.go:221] ["0 insert success: 1100"] [2025/04/06 08:28:08.169 +08:00] [INFO] [main.go:221] ["1 insert success: 1100"] [2025/04/06 08:28:08.292 +08:00] [INFO] [main.go:235] ["0 delete success: 600"] [2025/04/06 08:28:08.293 +08:00] [INFO] [main.go:221] ["0 insert success: 1200"] [2025/04/06 08:28:08.408 +08:00] [INFO] [main.go:221] ["1 insert success: 1200"] [2025/04/06 08:28:08.429 +08:00] [INFO] [main.go:235] ["1 delete success: 600"] table consistent_replicate_storage_s3.check1 not exists for 5-th check, retry later [WARN] --tz is deprecated in changefeed settings. Create changefeed successfully! ID: custom-changefeed-name Info: {"upstream_id":7489990252901441927,"namespace":"default","id":"custom-changefeed-name","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-06T08:28:08.432842782+08:00","start_ts":457152735049940994,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457152735049940994,"checkpoint_ts":457152735049940994,"checkpoint_time":"2025-04-06 08:28:08.368"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... [2025/04/06 08:28:08.529 +08:00] [INFO] [main.go:221] ["0 insert success: 1300"] [2025/04/06 08:28:08.653 +08:00] [INFO] [main.go:221] ["1 insert success: 1300"] [2025/04/06 08:28:08.768 +08:00] [INFO] [main.go:221] ["0 insert success: 1400"] [2025/04/06 08:28:08.771 +08:00] [INFO] [main.go:235] ["0 delete success: 700"] [2025/04/06 08:28:08.893 +08:00] [INFO] [main.go:221] ["1 insert success: 1400"] [2025/04/06 08:28:08.918 +08:00] [INFO] [main.go:235] ["1 delete success: 700"] [2025/04/06 08:28:09.013 +08:00] [INFO] [main.go:221] ["0 insert success: 1500"] [2025/04/06 08:28:09.135 +08:00] [INFO] [main.go:221] ["1 insert success: 1500"] [2025/04/06 08:28:09.258 +08:00] [INFO] [main.go:221] ["0 insert success: 1600"] [2025/04/06 08:28:09.262 +08:00] [INFO] [main.go:235] ["0 delete success: 800"] [2025/04/06 08:28:09.356 +08:00] [INFO] [main.go:221] ["1 insert success: 1600"] [2025/04/06 08:28:09.383 +08:00] [INFO] [main.go:235] ["1 delete success: 800"] [2025/04/06 08:28:09.505 +08:00] [INFO] [main.go:221] ["0 insert success: 1700"] =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/split_region/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... [2025/04/06 08:28:09.613 +08:00] [INFO] [main.go:221] ["1 insert success: 1700"] [2025/04/06 08:28:09.751 +08:00] [INFO] [main.go:221] ["0 insert success: 1800"] [2025/04/06 08:28:09.759 +08:00] [INFO] [main.go:235] ["0 delete success: 900"] + kill_tidb ++ ps aux ++ grep tidb-server ++ grep /tmp/tidb_cdc_test/synced_status_with_redo + info='jenkins 15297 4.6 0.0 2806852 190796 ? Sl 08:27 0:00 tidb-server -P 4000 -config /tmp/tidb_cdc_test/synced_status_with_redo/tidb-config-1743899270902462254.toml --store tikv --path 127.0.0.1:2379 --status=10080 --log-file /tmp/tidb_cdc_test/synced_status_with_redo/tidb.log jenkins 15301 10.5 0.0 2881416 214568 ? Sl 08:27 0:02 tidb-server -P 4001 -config /tmp/tidb_cdc_test/synced_status_with_redo/tidb-config-1743899270905495032.toml --store tikv --path 127.0.0.1:2379 --status=10081 --log-file /tmp/tidb_cdc_test/synced_status_with_redo/tidb_other.log jenkins 15375 10.8 0.0 2799584 222948 ? Sl 08:27 0:02 tidb-server -P 3306 -config /tmp/tidb_cdc_test/synced_status_with_redo/tidb-config-1743899270966049166.toml --store tikv --path 127.0.0.1:2479 --status=20080 --log-file /tmp/tidb_cdc_test/synced_status_with_redo/tidb_down.log' ++ ps aux ++ grep tidb-server ++ grep /tmp/tidb_cdc_test/synced_status_with_redo ++ awk '{print $2}' ++ xargs kill -9 ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 243 100 243 0 0 6035 0 --:--:-- --:--:-- --:--:-- 6075 + synced_status='{"synced":false,"sink_checkpoint_ts":"2025-04-06 08:28:07.980","puller_resolved_ts":"2025-04-06 08:28:03.930","last_synced_ts":"2025-04-06 08:28:04.430","now_ts":"2025-04-06 08:28:09.000","info":"The data syncing is not finished, please wait"}' ++ echo '{"synced":false,"sink_checkpoint_ts":"2025-04-06' '08:28:07.980","puller_resolved_ts":"2025-04-06' '08:28:03.930","last_synced_ts":"2025-04-06' '08:28:04.430","now_ts":"2025-04-06' '08:28:09.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq .synced + status=false + '[' false '!=' false ']' ++ echo '{"synced":false,"sink_checkpoint_ts":"2025-04-06' '08:28:07.980","puller_resolved_ts":"2025-04-06' '08:28:03.930","last_synced_ts":"2025-04-06' '08:28:04.430","now_ts":"2025-04-06' '08:28:09.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq -r .info + info='The data syncing is not finished, please wait' + target_message='The data syncing is not finished, please wait' + '[' 'The data syncing is not finished, please wait' '!=' 'The data syncing is not finished, please wait' ']' + sleep 130 + set +x [2025/04/06 08:28:09.841 +08:00] [INFO] [main.go:221] ["1 insert success: 1800"] [2025/04/06 08:28:09.869 +08:00] [INFO] [main.go:235] ["1 delete success: 900"] [2025/04/06 08:28:09.989 +08:00] [INFO] [main.go:221] ["0 insert success: 1900"] [2025/04/06 08:28:10.050 +08:00] [INFO] [main.go:221] ["1 insert success: 1900"] [2025/04/06 08:28:10.232 +08:00] [INFO] [main.go:221] ["0 insert success: 2000"] [2025/04/06 08:28:10.241 +08:00] [INFO] [main.go:235] ["0 delete success: 1000"] [2025/04/06 08:28:10.268 +08:00] [INFO] [main.go:221] ["1 insert success: 2000"] [2025/04/06 08:28:10.294 +08:00] [INFO] [main.go:235] ["1 delete success: 1000"] table consistent_replicate_storage_s3.check1 not exists for 6-th check, retry later [2025/04/06 08:28:10.470 +08:00] [INFO] [main.go:221] ["0 insert success: 2100"] [2025/04/06 08:28:10.482 +08:00] [INFO] [main.go:221] ["1 insert success: 2100"] table consistent_replicate_storage_s3.check1 not exists for 7-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/split_region Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... + endpoints=https://127.0.0.1:2579 + changefeed_id=custom-changefeed-name + expected_state=normal + error_msg=null + tls_dir=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates + [[ https://127.0.0.1:2579 =~ https ]] ++ cdc cli changefeed query --ca=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client.pem --key=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/_certificates/client-key.pem --pd=https://127.0.0.1:2579 -c custom-changefeed-name -s + info='{ "upstream_id": 7489990252901441927, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 457152735967707138, "checkpoint_time": "2025-04-06 08:28:11.869", "error": null }' + echo '{ "upstream_id": 7489990252901441927, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 457152735967707138, "checkpoint_time": "2025-04-06 08:28:11.869", "error": null }' { "upstream_id": 7489990252901441927, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 457152735967707138, "checkpoint_time": "2025-04-06 08:28:11.869", "error": null } ++ echo '{' '"upstream_id":' 7489990252901441927, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 457152735967707138, '"checkpoint_time":' '"2025-04-06' '08:28:11.869",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7489990252901441927, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 457152735967707138, '"checkpoint_time":' '"2025-04-06' '08:28:11.869",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_tls_with_auth.cli.44507.out cli unsafe delete-service-gc-safepoint Confirm that you know what this command will do and use it at your own risk [Y/N] CDC service GC safepoint truncated in PD! PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... table consistent_replicate_storage_s3.check1 not exists for 8-th check, retry later + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_tls_with_auth.cli.44534.out cli unsafe reset --no-confirm --pd=https://127.0.0.1:2579 reset and all metadata truncated in PD! PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... table consistent_replicate_storage_s3.check1 not exists for 9-th check, retry later [2025/04/06 08:28:16.403 +08:00] [INFO] [main.go:86] ["running ddl test: 2 addDropColumnDDL"] [2025/04/06 08:28:16.661 +08:00] [INFO] [main.go:221] ["0 insert success: 100"] [2025/04/06 08:28:16.671 +08:00] [INFO] [main.go:221] ["1 insert success: 100"] [2025/04/06 08:28:16.907 +08:00] [INFO] [main.go:221] ["0 insert success: 200"] [2025/04/06 08:28:16.909 +08:00] [INFO] [main.go:235] ["0 delete success: 100"] [2025/04/06 08:28:16.920 +08:00] [INFO] [main.go:235] ["1 delete success: 100"] [2025/04/06 08:28:16.921 +08:00] [INFO] [main.go:221] ["1 insert success: 200"] Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release [2025/04/06 08:28:17.158 +08:00] [INFO] [main.go:221] ["1 insert success: 300"] [2025/04/06 08:28:17.162 +08:00] [INFO] [main.go:221] ["0 insert success: 300"] + set +x [2025/04/06 08:28:17.550 +08:00] [INFO] [main.go:235] ["1 delete success: 200"] [2025/04/06 08:28:17.551 +08:00] [INFO] [main.go:221] ["1 insert success: 400"] [2025/04/06 08:28:17.556 +08:00] [INFO] [main.go:221] ["0 insert success: 400"] [2025/04/06 08:28:17.558 +08:00] [INFO] [main.go:235] ["0 delete success: 200"] [2025/04/06 08:28:17.786 +08:00] [INFO] [main.go:221] ["1 insert success: 500"] [2025/04/06 08:28:17.788 +08:00] [INFO] [main.go:221] ["0 insert success: 500"] [2025/04/06 08:28:18.005 +08:00] [INFO] [main.go:235] ["1 delete success: 300"] [2025/04/06 08:28:18.006 +08:00] [INFO] [main.go:221] ["1 insert success: 600"] [2025/04/06 08:28:18.016 +08:00] [INFO] [main.go:221] ["0 insert success: 600"] [2025/04/06 08:28:18.018 +08:00] [INFO] [main.go:235] ["0 delete success: 300"] [2025/04/06 08:28:18.228 +08:00] [INFO] [main.go:221] ["1 insert success: 700"] [2025/04/06 08:28:18.238 +08:00] [INFO] [main.go:221] ["0 insert success: 700"] Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... [2025/04/06 08:28:18.439 +08:00] [INFO] [main.go:235] ["1 delete success: 400"] [2025/04/06 08:28:18.440 +08:00] [INFO] [main.go:221] ["1 insert success: 800"] [2025/04/06 08:28:18.460 +08:00] [INFO] [main.go:221] ["0 insert success: 800"] [2025/04/06 08:28:18.463 +08:00] [INFO] [main.go:235] ["0 delete success: 400"] Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2025/04/06 08:28:18.649 +08:00] [INFO] [main.go:221] ["1 insert success: 900"] [2025/04/06 08:28:18.683 +08:00] [INFO] [main.go:221] ["0 insert success: 900"] [2025/04/06 08:28:18.862 +08:00] [INFO] [main.go:235] ["1 delete success: 500"] [2025/04/06 08:28:18.863 +08:00] [INFO] [main.go:221] ["1 insert success: 1000"] [2025/04/06 08:28:18.901 +08:00] [INFO] [main.go:221] ["0 insert success: 1000"] [2025/04/06 08:28:18.903 +08:00] [INFO] [main.go:235] ["0 delete success: 500"] [2025/04/06 08:28:19.078 +08:00] [INFO] [main.go:221] ["1 insert success: 1100"] table consistent_replicate_storage_s3.check1 not exists for 10-th check, retry later [2025/04/06 08:28:19.123 +08:00] [INFO] [main.go:221] ["0 insert success: 1100"] [2025/04/06 08:28:19.432 +08:00] [INFO] [main.go:235] ["1 delete success: 600"] [2025/04/06 08:28:19.433 +08:00] [INFO] [main.go:221] ["1 insert success: 1200"] [2025/04/06 08:28:19.491 +08:00] [INFO] [main.go:221] ["0 insert success: 1200"] [2025/04/06 08:28:19.493 +08:00] [INFO] [main.go:235] ["0 delete success: 600"] [2025/04/06 08:28:19.663 +08:00] [INFO] [main.go:221] ["1 insert success: 1300"] [2025/04/06 08:28:19.721 +08:00] [INFO] [main.go:221] ["0 insert success: 1300"] [2025/04/06 08:28:19.891 +08:00] [INFO] [main.go:235] ["1 delete success: 700"] [2025/04/06 08:28:19.892 +08:00] [INFO] [main.go:221] ["1 insert success: 1400"] [2025/04/06 08:28:19.949 +08:00] [INFO] [main.go:221] ["0 insert success: 1400"] [2025/04/06 08:28:19.952 +08:00] [INFO] [main.go:235] ["0 delete success: 700"] [2025/04/06 08:28:20.106 +08:00] [INFO] [main.go:221] ["1 insert success: 1500"] [2025/04/06 08:28:20.166 +08:00] [INFO] [main.go:221] ["0 insert success: 1500"] [2025/04/06 08:28:20.314 +08:00] [INFO] [main.go:235] ["1 delete success: 800"] [2025/04/06 08:28:20.315 +08:00] [INFO] [main.go:221] ["1 insert success: 1600"] [2025/04/06 08:28:20.380 +08:00] [INFO] [main.go:221] ["0 insert success: 1600"] [2025/04/06 08:28:20.382 +08:00] [INFO] [main.go:235] ["0 delete success: 800"] [2025/04/06 08:28:20.526 +08:00] [INFO] [main.go:221] ["1 insert success: 1700"] [2025/04/06 08:28:20.600 +08:00] [INFO] [main.go:221] ["0 insert success: 1700"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_tls_with_auth.cli.44631.out cli unsafe resolve-lock --region=40 [2025/04/06 08:28:20.748 +08:00] [INFO] [main.go:235] ["1 delete success: 900"] [2025/04/06 08:28:20.749 +08:00] [INFO] [main.go:221] ["1 insert success: 1800"] [2025/04/06 08:28:20.819 +08:00] [INFO] [main.go:221] ["0 insert success: 1800"] [2025/04/06 08:28:20.821 +08:00] [INFO] [main.go:235] ["0 delete success: 900"] PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... table consistent_replicate_storage_s3.check1 not exists for 11-th check, retry later [2025/04/06 08:28:20.959 +08:00] [INFO] [main.go:221] ["1 insert success: 1900"] [2025/04/06 08:28:21.023 +08:00] [INFO] [main.go:221] ["0 insert success: 1900"] [2025/04/06 08:28:21.179 +08:00] [INFO] [main.go:235] ["1 delete success: 1000"] [2025/04/06 08:28:21.181 +08:00] [INFO] [main.go:221] ["1 insert success: 2000"] [2025/04/06 08:28:21.345 +08:00] [INFO] [main.go:221] ["0 insert success: 2000"] [2025/04/06 08:28:21.348 +08:00] [INFO] [main.go:235] ["0 delete success: 1000"] + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_tls_with_auth.cli.44664.out cli unsafe resolve-lock --region=40 --ts=457152737409236996 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... table consistent_replicate_storage_s3.check1 not exists for 12-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489990443147436691 TiDB Cluster ID. tikv_gc_leader_uuid 65821fe4efc0003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-v-nsng0, pid:22611, start at 2025-04-06 08:28:20.83072234 +0800 CST m=+2.330551917 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:30:20.839 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:28:20.799 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:18:20.799 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489990443147436691 TiDB Cluster ID. tikv_gc_leader_uuid 65821fe4efc0003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-v-nsng0, pid:22611, start at 2025-04-06 08:28:20.83072234 +0800 CST m=+2.330551917 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:30:20.839 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:28:20.799 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:18:20.799 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489990441727518536 TiDB Cluster ID. tikv_gc_leader_uuid 65821fe4e24000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-v-nsng0, pid:22687, start at 2025-04-06 08:28:20.783017128 +0800 CST m=+2.208637488 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:30:20.830 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:28:20.796 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:18:20.796 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/split_region/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/split_region/tiflash/log/error.log arg matches is ArgMatches { args: {"memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/split_region/tiflash/log/proxy.log"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/split_region/tiflash-proxy.toml"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/split_region/tiflash/db/proxy"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + set +x % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 218 100 212 100 6 1797 50 --:--:-- --:--:-- --:--:-- 1811 { "error_msg": "[CDC:ErrAPIInvalidParam]invalid log level: json: cannot unmarshal string into Go value of type struct { Level string \"json:\\\"log_level\\\"\" }", "error_code": "CDC:ErrAPIInvalidParam" table consistent_replicate_storage_s3.check1 not exists for 13-th check, retry later table changefeed_dup_error_restart.finish_mark_2 exists check diff successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Sun Apr 6 08:28:22 CST 2025] <<<<<< run test case changefeed_dup_error_restart success! >>>>>> + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.split_region.cli.24039.out cli tso query --pd=http://127.0.0.1:2379 table consistent_replicate_storage_s3.check1 not exists for 14-th check, retry later + set +x + tso='457152739462086657 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 457152739462086657 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Sun Apr 6 08:28:26 CST 2025] <<<<<< START cdc server in split_region case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + GO_FAILPOINTS= + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.split_region.2408524087.out server --log-file /tmp/tidb_cdc_test/split_region/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/split_region/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 } % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 182 100 182 0 0 1543 0 --:--:-- --:--:-- --:--:-- 1555 { "version": "v9.0.0-beta.1-9-g4c631d595", "git_hash": "4c631d5951854ac91765b729fe3849d8e8d5ed8a", "id": "3ec91ee3-5d19-4760-9771-b1cc797e1306", "pid": 43831, "is_owner": true [2025/04/06 08:28:27.286 +08:00] [INFO] [main.go:86] ["running ddl test: 3 addDropColumnDDL2"] [2025/04/06 08:28:27.525 +08:00] [INFO] [main.go:221] ["1 insert success: 100"] [2025/04/06 08:28:27.538 +08:00] [INFO] [main.go:221] ["0 insert success: 100"] [2025/04/06 08:28:27.727 +08:00] [INFO] [main.go:235] ["1 delete success: 100"] [2025/04/06 08:28:27.728 +08:00] [INFO] [main.go:221] ["1 insert success: 200"] [2025/04/06 08:28:27.741 +08:00] [INFO] [main.go:221] ["0 insert success: 200"] [2025/04/06 08:28:27.743 +08:00] [INFO] [main.go:235] ["0 delete success: 100"] }wait process cdc.test exit for 1-th time... [2025/04/06 08:28:27.921 +08:00] [INFO] [main.go:221] ["1 insert success: 300"] [2025/04/06 08:28:27.931 +08:00] [INFO] [main.go:221] ["0 insert success: 300"] [2025/04/06 08:28:28.119 +08:00] [INFO] [main.go:235] ["1 delete success: 200"] [2025/04/06 08:28:28.120 +08:00] [INFO] [main.go:221] ["1 insert success: 400"] [2025/04/06 08:28:28.132 +08:00] [INFO] [main.go:221] ["0 insert success: 400"] [2025/04/06 08:28:28.135 +08:00] [INFO] [main.go:235] ["0 delete success: 200"] [2025/04/06 08:28:28.337 +08:00] [INFO] [main.go:221] ["1 insert success: 500"] [2025/04/06 08:28:28.354 +08:00] [INFO] [main.go:221] ["0 insert success: 500"] wait process cdc.test exit for 2-th time... [2025/04/06 08:28:28.536 +08:00] [INFO] [main.go:235] ["1 delete success: 300"] [2025/04/06 08:28:28.537 +08:00] [INFO] [main.go:221] ["1 insert success: 600"] [2025/04/06 08:28:28.555 +08:00] [INFO] [main.go:221] ["0 insert success: 600"] [2025/04/06 08:28:28.557 +08:00] [INFO] [main.go:235] ["0 delete success: 300"] cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Sun Apr 6 08:28:28 CST 2025] <<<<<< run test case cli_tls_with_auth success! >>>>>> [2025/04/06 08:28:28.742 +08:00] [INFO] [main.go:221] ["1 insert success: 700"] [2025/04/06 08:28:28.763 +08:00] [INFO] [main.go:221] ["0 insert success: 700"] [2025/04/06 08:28:28.949 +08:00] [INFO] [main.go:235] ["1 delete success: 400"] [2025/04/06 08:28:28.951 +08:00] [INFO] [main.go:221] ["1 insert success: 800"] [2025/04/06 08:28:28.974 +08:00] [INFO] [main.go:221] ["0 insert success: 800"] [2025/04/06 08:28:28.976 +08:00] [INFO] [main.go:235] ["0 delete success: 400"] [2025/04/06 08:28:29.152 +08:00] [INFO] [main.go:221] ["1 insert success: 900"] table consistent_replicate_storage_s3.check1 exists table consistent_replicate_storage_s3.t2 not exists for 1-th check, retry later [2025/04/06 08:28:29.183 +08:00] [INFO] [main.go:221] ["0 insert success: 900"] [2025/04/06 08:28:29.369 +08:00] [INFO] [main.go:235] ["1 delete success: 500"] [2025/04/06 08:28:29.370 +08:00] [INFO] [main.go:221] ["1 insert success: 1000"] [2025/04/06 08:28:29.401 +08:00] [INFO] [main.go:221] ["0 insert success: 1000"] [2025/04/06 08:28:29.404 +08:00] [INFO] [main.go:235] ["0 delete success: 500"] [2025/04/06 08:28:29.588 +08:00] [INFO] [main.go:221] ["1 insert success: 1100"] [2025/04/06 08:28:29.618 +08:00] [INFO] [main.go:221] ["0 insert success: 1100"] [2025/04/06 08:28:29.799 +08:00] [INFO] [main.go:235] ["1 delete success: 600"] [2025/04/06 08:28:29.801 +08:00] [INFO] [main.go:221] ["1 insert success: 1200"] [2025/04/06 08:28:29.845 +08:00] [INFO] [main.go:221] ["0 insert success: 1200"] [2025/04/06 08:28:29.847 +08:00] [INFO] [main.go:235] ["0 delete success: 600"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 06 Apr 2025 00:28:29 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5f8e35a1-891f-473e-841f-13699317d4e1 {"id":"5f8e35a1-891f-473e-841f-13699317d4e1","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899307} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087f72e8f7 5f8e35a1-891f-473e-841f-13699317d4e1 /tidb/cdc/default/default/upstream/7489990443147436691 {"id":7489990443147436691,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5f8e35a1-891f-473e-841f-13699317d4e1 {"id":"5f8e35a1-891f-473e-841f-13699317d4e1","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899307} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087f72e8f7 5f8e35a1-891f-473e-841f-13699317d4e1 /tidb/cdc/default/default/upstream/7489990443147436691 {"id":7489990443147436691,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5f8e35a1-891f-473e-841f-13699317d4e1 {"id":"5f8e35a1-891f-473e-841f-13699317d4e1","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899307} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087f72e8f7 5f8e35a1-891f-473e-841f-13699317d4e1 /tidb/cdc/default/default/upstream/7489990443147436691 {"id":7489990443147436691,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.split_region.cli.24152.out cli changefeed create --start-ts=457152739462086657 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ --config /tmp/tidb_cdc_test/split_region/pulsar_test.toml [2025/04/06 08:28:30.012 +08:00] [INFO] [main.go:221] ["1 insert success: 1300"] [2025/04/06 08:28:30.057 +08:00] [INFO] [main.go:221] ["0 insert success: 1300"] Create changefeed successfully! ID: 2919cb5d-3aca-433c-a37d-a12f2bbcc7a7 Info: {"upstream_id":7489990443147436691,"namespace":"default","id":"2919cb5d-3aca-433c-a37d-a12f2bbcc7a7","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-06T08:28:30.191839177+08:00","start_ts":457152739462086657,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457152739462086657,"checkpoint_ts":457152739462086657,"checkpoint_time":"2025-04-06 08:28:25.199"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... [2025/04/06 08:28:30.231 +08:00] [INFO] [main.go:235] ["1 delete success: 700"] [2025/04/06 08:28:30.232 +08:00] [INFO] [main.go:221] ["1 insert success: 1400"] [2025/04/06 08:28:30.272 +08:00] [INFO] [main.go:221] ["0 insert success: 1400"] [2025/04/06 08:28:30.275 +08:00] [INFO] [main.go:235] ["0 delete success: 700"] [2025/04/06 08:28:30.454 +08:00] [INFO] [main.go:221] ["1 insert success: 1500"] [2025/04/06 08:28:30.501 +08:00] [INFO] [main.go:221] ["0 insert success: 1500"] [2025/04/06 08:28:30.656 +08:00] [INFO] [main.go:235] ["1 delete success: 800"] [2025/04/06 08:28:30.657 +08:00] [INFO] [main.go:221] ["1 insert success: 1600"] [2025/04/06 08:28:30.721 +08:00] [INFO] [main.go:221] ["0 insert success: 1600"] [2025/04/06 08:28:30.723 +08:00] [INFO] [main.go:235] ["0 delete success: 800"] [2025/04/06 08:28:30.865 +08:00] [INFO] [main.go:221] ["1 insert success: 1700"] [2025/04/06 08:28:30.940 +08:00] [INFO] [main.go:221] ["0 insert success: 1700"] table consistent_replicate_storage_s3.t2 not exists for 2-th check, retry later [2025/04/06 08:28:31.077 +08:00] [INFO] [main.go:235] ["1 delete success: 900"] [2025/04/06 08:28:31.078 +08:00] [INFO] [main.go:221] ["1 insert success: 1800"] [2025/04/06 08:28:31.180 +08:00] [INFO] [main.go:221] ["0 insert success: 1800"] [2025/04/06 08:28:31.182 +08:00] [INFO] [main.go:235] ["0 delete success: 900"] [2025/04/06 08:28:31.354 +08:00] [INFO] [main.go:221] ["1 insert success: 1900"] [2025/04/06 08:28:31.537 +08:00] [INFO] [main.go:221] ["0 insert success: 1900"] [2025/04/06 08:28:31.619 +08:00] [INFO] [main.go:235] ["1 delete success: 1000"] [2025/04/06 08:28:31.621 +08:00] [INFO] [main.go:221] ["1 insert success: 2000"] + set +x table split_region.test1 not exists for 1-th check, retry later [2025/04/06 08:28:31.778 +08:00] [INFO] [main.go:221] ["0 insert success: 2000"] [2025/04/06 08:28:31.781 +08:00] [INFO] [main.go:235] ["0 delete success: 1000"] [2025/04/06 08:28:31.846 +08:00] [INFO] [main.go:221] ["1 insert success: 2100"] [2025/04/06 08:28:32.109 +08:00] [INFO] [main.go:221] ["0 insert success: 2100"] [2025/04/06 08:28:32.164 +08:00] [INFO] [main.go:235] ["1 delete success: 1100"] [2025/04/06 08:28:32.166 +08:00] [INFO] [main.go:221] ["1 insert success: 2200"] table consistent_replicate_storage_s3.t2 not exists for 3-th check, retry later table split_region.test1 exists table split_region.test2 not exists for 1-th check, retry later table consistent_replicate_storage_s3.t2 not exists for 4-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/safe_mode/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... table split_region.test2 exists check diff successfully table consistent_replicate_storage_s3.t2 not exists for 5-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/safe_mode Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... [2025/04/06 08:28:38.161 +08:00] [INFO] [main.go:86] ["running ddl test: 4 modifyColumnDDL"] [2025/04/06 08:28:38.398 +08:00] [INFO] [main.go:221] ["1 insert success: 100"] [2025/04/06 08:28:38.420 +08:00] [INFO] [main.go:221] ["0 insert success: 100"] [2025/04/06 08:28:38.690 +08:00] [INFO] [main.go:221] ["1 insert success: 200"] [2025/04/06 08:28:38.692 +08:00] [INFO] [main.go:235] ["1 delete success: 100"] [2025/04/06 08:28:38.717 +08:00] [INFO] [main.go:221] ["0 insert success: 200"] [2025/04/06 08:28:38.719 +08:00] [INFO] [main.go:235] ["0 delete success: 100"] [2025/04/06 08:28:38.884 +08:00] [INFO] [main.go:221] ["1 insert success: 300"] [2025/04/06 08:28:38.925 +08:00] [INFO] [main.go:221] ["0 insert success: 300"] table consistent_replicate_storage_s3.t2 not exists for 6-th check, retry later [2025/04/06 08:28:39.092 +08:00] [INFO] [main.go:221] ["1 insert success: 400"] [2025/04/06 08:28:39.094 +08:00] [INFO] [main.go:235] ["1 delete success: 200"] [2025/04/06 08:28:39.134 +08:00] [INFO] [main.go:221] ["0 insert success: 400"] [2025/04/06 08:28:39.136 +08:00] [INFO] [main.go:235] ["0 delete success: 200"] [2025/04/06 08:28:39.293 +08:00] [INFO] [main.go:221] ["1 insert success: 500"] [2025/04/06 08:28:39.333 +08:00] [INFO] [main.go:221] ["0 insert success: 500"] [2025/04/06 08:28:39.488 +08:00] [INFO] [main.go:221] ["1 insert success: 600"] [2025/04/06 08:28:39.490 +08:00] [INFO] [main.go:235] ["1 delete success: 300"] [2025/04/06 08:28:39.533 +08:00] [INFO] [main.go:221] ["0 insert success: 600"] [2025/04/06 08:28:39.535 +08:00] [INFO] [main.go:235] ["0 delete success: 300"] [2025/04/06 08:28:39.683 +08:00] [INFO] [main.go:221] ["1 insert success: 700"] [2025/04/06 08:28:39.724 +08:00] [INFO] [main.go:221] ["0 insert success: 700"] [2025/04/06 08:28:39.871 +08:00] [INFO] [main.go:221] ["1 insert success: 800"] [2025/04/06 08:28:39.873 +08:00] [INFO] [main.go:235] ["1 delete success: 400"] [2025/04/06 08:28:39.915 +08:00] [INFO] [main.go:221] ["0 insert success: 800"] [2025/04/06 08:28:39.917 +08:00] [INFO] [main.go:235] ["0 delete success: 400"] [2025/04/06 08:28:40.077 +08:00] [INFO] [main.go:221] ["1 insert success: 900"] [2025/04/06 08:28:40.123 +08:00] [INFO] [main.go:221] ["0 insert success: 900"] [2025/04/06 08:28:40.362 +08:00] [INFO] [main.go:221] ["1 insert success: 1000"] [2025/04/06 08:28:40.365 +08:00] [INFO] [main.go:235] ["1 delete success: 500"] [2025/04/06 08:28:40.461 +08:00] [INFO] [main.go:221] ["0 insert success: 1000"] [2025/04/06 08:28:40.464 +08:00] [INFO] [main.go:235] ["0 delete success: 500"] [2025/04/06 08:28:40.618 +08:00] [INFO] [main.go:221] ["1 insert success: 1100"] [2025/04/06 08:28:40.666 +08:00] [INFO] [main.go:221] ["0 insert success: 1100"] =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/debezium_basic/run.sh using Sink-Type: mysql... <<================= [Sun Apr 6 08:28:39 CST 2025] <<<<<< run test case debezium_basic success! >>>>>> [2025/04/06 08:28:40.831 +08:00] [INFO] [main.go:221] ["1 insert success: 1200"] [2025/04/06 08:28:40.833 +08:00] [INFO] [main.go:235] ["1 delete success: 600"] [2025/04/06 08:28:40.875 +08:00] [INFO] [main.go:221] ["0 insert success: 1200"] [2025/04/06 08:28:40.878 +08:00] [INFO] [main.go:235] ["0 delete success: 600"] table consistent_replicate_storage_s3.t2 exists [2025/04/06 08:28:41.182 +08:00] [INFO] [main.go:221] ["1 insert success: 1300"] [2025/04/06 08:28:41.263 +08:00] [INFO] [main.go:221] ["0 insert success: 1300"] [2025/04/06 08:28:41.429 +08:00] [INFO] [main.go:221] ["1 insert success: 1400"] [2025/04/06 08:28:41.431 +08:00] [INFO] [main.go:235] ["1 delete success: 700"] [2025/04/06 08:28:41.474 +08:00] [INFO] [main.go:221] ["0 insert success: 1400"] [2025/04/06 08:28:41.476 +08:00] [INFO] [main.go:235] ["0 delete success: 700"] check diff successfully [2025/04/06 08:28:41.632 +08:00] [INFO] [main.go:221] ["1 insert success: 1500"] [2025/04/06 08:28:41.688 +08:00] [INFO] [main.go:221] ["0 insert success: 1500"] [2025/04/06 08:28:41.988 +08:00] [INFO] [main.go:221] ["1 insert success: 1600"] [2025/04/06 08:28:41.991 +08:00] [INFO] [main.go:235] ["1 delete success: 800"] wait process cdc.test exit for 1-th time... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release [2025/04/06 08:28:42.054 +08:00] [INFO] [main.go:221] ["0 insert success: 1600"] [2025/04/06 08:28:42.057 +08:00] [INFO] [main.go:235] ["0 delete success: 800"] [2025/04/06 08:28:42.205 +08:00] [INFO] [main.go:221] ["1 insert success: 1700"] [2025/04/06 08:28:42.261 +08:00] [INFO] [main.go:221] ["0 insert success: 1700"] [2025/04/06 08:28:42.439 +08:00] [INFO] [main.go:221] ["1 insert success: 1800"] [2025/04/06 08:28:42.441 +08:00] [INFO] [main.go:235] ["1 delete success: 900"] wait process cdc.test exit for 2-th time... [2025/04/06 08:28:42.644 +08:00] [INFO] [main.go:221] ["0 insert success: 1800"] [2025/04/06 08:28:42.646 +08:00] [INFO] [main.go:235] ["0 delete success: 900"] [2025/04/06 08:28:42.793 +08:00] [INFO] [main.go:221] ["1 insert success: 1900"] wait process cdc.test exit for 3-th time... [2025/04/06 08:28:42.851 +08:00] [INFO] [main.go:221] ["0 insert success: 1900"] [2025/04/06 08:28:43.000 +08:00] [INFO] [main.go:221] ["1 insert success: 2000"] [2025/04/06 08:28:43.002 +08:00] [INFO] [main.go:235] ["1 delete success: 1000"] [2025/04/06 08:28:43.050 +08:00] [INFO] [main.go:221] ["0 insert success: 2000"] [2025/04/06 08:28:43.052 +08:00] [INFO] [main.go:235] ["0 delete success: 1000"] =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/kafka_simple_basic/run.sh using Sink-Type: mysql... <<================= [Sun Apr 6 08:28:43 CST 2025] <<<<<< run test case kafka_simple_basic success! >>>>>> cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Sun Apr 6 08:28:43 CST 2025] <<<<<< START cdc server in consistent_replicate_storage_s3 case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sink/dmlsink/txn/mysql/MySQLSinkHangLongTime=return(true)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.consistent_replicate_storage_s3.1766317665.out server --log-file /tmp/tidb_cdc_test/consistent_replicate_storage_s3/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/consistent_replicate_storage_s3/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 06 Apr 2025 00:28:46 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/8c6ad029-63cf-4f9d-8684-003eedf81071 {UpstreamID:7489990294254066684 Namespace:default ID:8c6ad029-63cf-4f9d-8684-003eedf81071 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-06 08:27:54.681140454 +0800 CST StartTs:457152731441004547 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc000f6e990 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457152731454111750} {CheckpointTs:457152732751462417 MinTableBarrierTs:457152744980480005 AdminJobType:noop} span: {tableID:116, startKey:7480000000000000ff745f720000000000fa, endKey:7480000000000000ff745f730000000000fa}, resolvedTs: 457152744718598150, checkpointTs: 457152744718598149, state: Replicating span: {tableID:119, startKey:7480000000000000ff775f720000000000fa, endKey:7480000000000000ff775f730000000000fa}, resolvedTs: 457152744718598149, checkpointTs: 457152744718598149, state: Replicating span: {tableID:121, startKey:7480000000000000ff795f720000000000fa, endKey:7480000000000000ff795f730000000000fa}, resolvedTs: 457152744718598149, checkpointTs: 457152744718598149, state: Replicating span: {tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}, resolvedTs: 457152744718598150, checkpointTs: 457152744718598149, state: Replicating span: {tableID:120, startKey:7480000000000000ff785f720000000000fa, endKey:7480000000000000ff785f730000000000fa}, resolvedTs: 457152744718598149, checkpointTs: 457152744718598149, state: Replicating span: {tableID:126, startKey:7480000000000000ff7e5f720000000000fa, endKey:7480000000000000ff7e5f730000000000fa}, resolvedTs: 457152744718598150, checkpointTs: 457152744718598149, state: Replicating span: {tableID:123, startKey:7480000000000000ff7b5f720000000000fa, endKey:7480000000000000ff7b5f730000000000fa}, resolvedTs: 457152744718598150, checkpointTs: 457152744718598149, state: Replicating span: {tableID:124, startKey:7480000000000000ff7c5f720000000000fa, endKey:7480000000000000ff7c5f730000000000fa}, resolvedTs: 457152744718598149, checkpointTs: 457152744718598149, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e8928b6b-e128-4df7-af82-33c0dc892b7e {"id":"e8928b6b-e128-4df7-af82-33c0dc892b7e","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899323} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087ef5ece1 e8928b6b-e128-4df7-af82-33c0dc892b7e /tidb/cdc/default/default/changefeed/info/8c6ad029-63cf-4f9d-8684-003eedf81071 {"upstream-id":7489990294254066684,"namespace":"default","changefeed-id":"8c6ad029-63cf-4f9d-8684-003eedf81071","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-06T08:27:54.681140454+08:00","start-ts":457152731441004547,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"s3://logbucket/test-changefeed?endpoint=http://127.0.0.1:24927/","use-file-backend":false,"compression":"","flush-concurrency":2,"memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457152731454111750} /tidb/cdc/default/default/changefeed/status/8c6ad029-63cf-4f9d-8684-003eedf81071 {"checkpoint-ts":457152744718598149,"min-table-barrier-ts":457152744980480005,"admin-job-type":0} /tidb/cdc/default/default/task/position/e8928b6b-e128-4df7-af82-33c0dc892b7e/8c6ad029-63cf-4f9d-8684-003eedf81071 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489990294254066684 {"id":7489990294254066684,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/8c6ad029-63cf-4f9d-8684-003eedf81071 {UpstreamID:7489990294254066684 Namespace:default ID:8c6ad029-63cf-4f9d-8684-003eedf81071 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-06 08:27:54.681140454 +0800 CST StartTs:457152731441004547 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc000f6e990 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457152731454111750} {CheckpointTs:457152732751462417 MinTableBarrierTs:457152744980480005 AdminJobType:noop} span: {tableID:116, startKey:7480000000000000ff745f720000000000fa, endKey:7480000000000000ff745f730000000000fa}, resolvedTs: 457152744718598150, checkpointTs: 457152744718598149, state: Replicating span: {tableID:119, startKey:7480000000000000ff775f720000000000fa, endKey:7480000000000000ff775f730000000000fa}, resolvedTs: 457152744718598149, checkpointTs: 457152744718598149, state: Replicating span: {tableID:121, startKey:7480000000000000ff795f720000000000fa, endKey:7480000000000000ff795f730000000000fa}, resolvedTs: 457152744718598149, checkpointTs: 457152744718598149, state: Replicating span: {tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}, resolvedTs: 457152744718598150, checkpointTs: 457152744718598149, state: Replicating span: {tableID:120, startKey:7480000000000000ff785f720000000000fa, endKey:7480000000000000ff785f730000000000fa}, resolvedTs: 457152744718598149, checkpointTs: 457152744718598149, state: Replicating span: {tableID:126, startKey:7480000000000000ff7e5f720000000000fa, endKey:7480000000000000ff7e5f730000000000fa}, resolvedTs: 457152744718598150, checkpointTs: 457152744718598149, state: Replicating span: {tableID:123, startKey:7480000000000000ff7b5f720000000000fa, endKey:7480000000000000ff7b5f730000000000fa}, resolvedTs: 457152744718598150, checkpointTs: 457152744718598149, state: Replicating span: {tableID:124, startKey:7480000000000000ff7c5f720000000000fa, endKey:7480000000000000ff7c5f730000000000fa}, resolvedTs: 457152744718598149, checkpointTs: 457152744718598149, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e8928b6b-e128-4df7-af82-33c0dc892b7e {"id":"e8928b6b-e128-4df7-af82-33c0dc892b7e","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899323} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087ef5ece1 e8928b6b-e128-4df7-af82-33c0dc892b7e /tidb/cdc/default/default/changefeed/info/8c6ad029-63cf-4f9d-8684-003eedf81071 {"upstream-id":7489990294254066684,"namespace":"default","changefeed-id":"8c6ad029-63cf-4f9d-8684-003eedf81071","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-06T08:27:54.681140454+08:00","start-ts":457152731441004547,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"s3://logbucket/test-changefeed?endpoint=http://127.0.0.1:24927/","use-file-backend":false,"compression":"","flush-concurrency":2,"memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457152731454111750} /tidb/cdc/default/default/changefeed/status/8c6ad029-63cf-4f9d-8684-003eedf81071 {"checkpoint-ts":457152744718598149,"min-table-barrier-ts":457152744980480005,"admin-job-type":0} /tidb/cdc/default/default/task/position/e8928b6b-e128-4df7-af82-33c0dc892b7e/8c6ad029-63cf-4f9d-8684-003eedf81071 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489990294254066684 {"id":7489990294254066684,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/8c6ad029-63cf-4f9d-8684-003eedf81071 {UpstreamID:7489990294254066684 Namespace:default ID:8c6ad029-63cf-4f9d-8684-003eedf81071 SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-06 08:27:54.681140454 +0800 CST StartTs:457152731441004547 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc000f6e990 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457152731454111750} {CheckpointTs:457152732751462417 MinTableBarrierTs:457152744980480005 AdminJobType:noop} span: {tableID:116, startKey:7480000000000000ff745f720000000000fa, endKey:7480000000000000ff745f730000000000fa}, resolvedTs: 457152744718598150, checkpointTs: 457152744718598149, state: Replicating span: {tableID:119, startKey:7480000000000000ff775f720000000000fa, endKey:7480000000000000ff775f730000000000fa}, resolvedTs: 457152744718598149, checkpointTs: 457152744718598149, state: Replicating span: {tableID:121, startKey:7480000000000000ff795f720000000000fa, endKey:7480000000000000ff795f730000000000fa}, resolvedTs: 457152744718598149, checkpointTs: 457152744718598149, state: Replicating span: {tableID:114, startKey:7480000000000000ff725f720000000000fa, endKey:7480000000000000ff725f730000000000fa}, resolvedTs: 457152744718598150, checkpointTs: 457152744718598149, state: Replicating span: {tableID:120, startKey:7480000000000000ff785f720000000000fa, endKey:7480000000000000ff785f730000000000fa}, resolvedTs: 457152744718598149, checkpointTs: 457152744718598149, state: Replicating span: {tableID:126, startKey:7480000000000000ff7e5f720000000000fa, endKey:7480000000000000ff7e5f730000000000fa}, resolvedTs: 457152744718598150, checkpointTs: 457152744718598149, state: Replicating span: {tableID:123, startKey:7480000000000000ff7b5f720000000000fa, endKey:7480000000000000ff7b5f730000000000fa}, resolvedTs: 457152744718598150, checkpointTs: 457152744718598149, state: Replicating span: {tableID:124, startKey:7480000000000000ff7c5f720000000000fa, endKey:7480000000000000ff7c5f730000000000fa}, resolvedTs: 457152744718598149, checkpointTs: 457152744718598149, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e8928b6b-e128-4df7-af82-33c0dc892b7e {"id":"e8928b6b-e128-4df7-af82-33c0dc892b7e","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899323} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087ef5ece1 e8928b6b-e128-4df7-af82-33c0dc892b7e /tidb/cdc/default/default/changefeed/info/8c6ad029-63cf-4f9d-8684-003eedf81071 {"upstream-id":7489990294254066684,"namespace":"default","changefeed-id":"8c6ad029-63cf-4f9d-8684-003eedf81071","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-06T08:27:54.681140454+08:00","start-ts":457152731441004547,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"s3://logbucket/test-changefeed?endpoint=http://127.0.0.1:24927/","use-file-backend":false,"compression":"","flush-concurrency":2,"memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457152731454111750} /tidb/cdc/default/default/changefeed/status/8c6ad029-63cf-4f9d-8684-003eedf81071 {"checkpoint-ts":457152744718598149,"min-table-barrier-ts":457152744980480005,"admin-job-type":0} /tidb/cdc/default/default/task/position/e8928b6b-e128-4df7-af82-33c0dc892b7e/8c6ad029-63cf-4f9d-8684-003eedf81071 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489990294254066684 {"id":7489990294254066684,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/kafka_simple_basic_avro/run.sh using Sink-Type: mysql... <<================= [Sun Apr 6 08:28:47 CST 2025] <<<<<< run test case kafka_simple_basic_avro success! >>>>>> check diff failed 1-th time, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489990551107061146 TiDB Cluster ID. tikv_gc_leader_uuid 65821ffdad80004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-9-92vm1, pid:20926, start at 2025-04-06 08:28:46.138921022 +0800 CST m=+2.255408650 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:30:46.146 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:28:46.134 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:18:46.134 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489990551107061146 TiDB Cluster ID. tikv_gc_leader_uuid 65821ffdad80004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-9-92vm1, pid:20926, start at 2025-04-06 08:28:46.138921022 +0800 CST m=+2.255408650 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:30:46.146 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:28:46.134 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:18:46.134 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489990546844333417 TiDB Cluster ID. tikv_gc_leader_uuid 65821ffdad80002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-9-92vm1, pid:20998, start at 2025-04-06 08:28:46.134876056 +0800 CST m=+2.185953719 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:30:46.143 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:28:46.134 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:18:46.134 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/safe_mode/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/safe_mode/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/safe_mode/tiflash-proxy.toml"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/safe_mode/tiflash/db/proxy"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/safe_mode/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [2025/04/06 08:28:49.073 +08:00] [INFO] [main.go:86] ["running ddl test: 5 addDropIndexDDL"] [2025/04/06 08:28:49.377 +08:00] [INFO] [main.go:221] ["1 insert success: 100"] [2025/04/06 08:28:49.381 +08:00] [INFO] [main.go:221] ["0 insert success: 100"] [2025/04/06 08:28:49.565 +08:00] [INFO] [main.go:221] ["1 insert success: 200"] [2025/04/06 08:28:49.567 +08:00] [INFO] [main.go:235] ["1 delete success: 100"] [2025/04/06 08:28:49.570 +08:00] [INFO] [main.go:221] ["0 insert success: 200"] [2025/04/06 08:28:49.572 +08:00] [INFO] [main.go:235] ["0 delete success: 100"] [2025/04/06 08:28:49.931 +08:00] [INFO] [main.go:221] ["1 insert success: 300"] [2025/04/06 08:28:49.941 +08:00] [INFO] [main.go:221] ["0 insert success: 300"] check diff successfully [2025/04/06 08:28:50.175 +08:00] [INFO] [main.go:221] ["1 insert success: 400"] [2025/04/06 08:28:50.177 +08:00] [INFO] [main.go:235] ["1 delete success: 200"] [2025/04/06 08:28:50.181 +08:00] [INFO] [main.go:221] ["0 insert success: 400"] [2025/04/06 08:28:50.183 +08:00] [INFO] [main.go:235] ["0 delete success: 200"] [2025/04/06 08:28:50.447 +08:00] [INFO] [main.go:221] ["1 insert success: 500"] [2025/04/06 08:28:50.458 +08:00] [INFO] [main.go:221] ["0 insert success: 500"] wait process cdc.test exit for 1-th time... [2025/04/06 08:28:50.834 +08:00] [INFO] [main.go:221] ["1 insert success: 600"] [2025/04/06 08:28:50.836 +08:00] [INFO] [main.go:235] ["1 delete success: 300"] [2025/04/06 08:28:50.849 +08:00] [INFO] [main.go:221] ["0 insert success: 600"] [2025/04/06 08:28:50.852 +08:00] [INFO] [main.go:235] ["0 delete success: 300"] wait process cdc.test exit for 2-th time... [2025/04/06 08:28:51.099 +08:00] [INFO] [main.go:221] ["1 insert success: 700"] [2025/04/06 08:28:51.140 +08:00] [INFO] [main.go:221] ["0 insert success: 700"] cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Sun Apr 6 08:28:51 CST 2025] <<<<<< run test case split_region success! >>>>>> [2025/04/06 08:28:51.555 +08:00] [INFO] [main.go:221] ["1 insert success: 800"] [2025/04/06 08:28:51.559 +08:00] [INFO] [main.go:235] ["1 delete success: 400"] [Sun Apr 6 08:28:51 CST 2025] <<<<<< START cdc server in safe_mode case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.safe_mode.2234622348.out server --log-file /tmp/tidb_cdc_test/safe_mode/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/safe_mode/cdc_data --cluster-id default + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/kafka_simple_handle_key_only/run.sh using Sink-Type: mysql... <<================= [Sun Apr 6 08:28:51 CST 2025] <<<<<< run test case kafka_simple_handle_key_only success! >>>>>> [2025/04/06 08:28:51.843 +08:00] [INFO] [main.go:221] ["0 insert success: 800"] [2025/04/06 08:28:51.848 +08:00] [INFO] [main.go:235] ["0 delete success: 400"] [2025/04/06 08:28:52.018 +08:00] [INFO] [main.go:221] ["1 insert success: 900"] [2025/04/06 08:28:52.578 +08:00] [INFO] [main.go:221] ["0 insert success: 900"] [2025/04/06 08:28:52.580 +08:00] [INFO] [main.go:221] ["1 insert success: 1000"] [2025/04/06 08:28:52.582 +08:00] [INFO] [main.go:235] ["1 delete success: 500"] [2025/04/06 08:28:52.842 +08:00] [INFO] [main.go:221] ["1 insert success: 1100"] [2025/04/06 08:28:53.049 +08:00] [INFO] [main.go:221] ["0 insert success: 1000"] [2025/04/06 08:28:53.053 +08:00] [INFO] [main.go:235] ["0 delete success: 500"] [2025/04/06 08:28:53.278 +08:00] [INFO] [main.go:221] ["1 insert success: 1200"] [2025/04/06 08:28:53.280 +08:00] [INFO] [main.go:235] ["1 delete success: 600"] [2025/04/06 08:28:53.527 +08:00] [INFO] [main.go:221] ["0 insert success: 1100"] [2025/04/06 08:28:53.638 +08:00] [INFO] [main.go:221] ["1 insert success: 1300"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 06 Apr 2025 00:28:54 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0b93f882-14e1-4955-882f-13977b518fd2 {"id":"0b93f882-14e1-4955-882f-13977b518fd2","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899331} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087fd613ed 0b93f882-14e1-4955-882f-13977b518fd2 /tidb/cdc/default/default/upstream/7489990551107061146 {"id":7489990551107061146,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0b93f882-14e1-4955-882f-13977b518fd2 {"id":"0b93f882-14e1-4955-882f-13977b518fd2","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899331} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087fd613ed 0b93f882-14e1-4955-882f-13977b518fd2 /tidb/cdc/default/default/upstream/7489990551107061146 {"id":7489990551107061146,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0b93f882-14e1-4955-882f-13977b518fd2 {"id":"0b93f882-14e1-4955-882f-13977b518fd2","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899331} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196087fd613ed 0b93f882-14e1-4955-882f-13977b518fd2 /tidb/cdc/default/default/upstream/7489990551107061146 {"id":7489990551107061146,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.safe_mode.cli.22405.out cli changefeed create '--sink-uri=mysql://normal:123456@127.0.0.1:3306/?safe-mode=true' Create changefeed successfully! ID: 99ffc8da-5352-4b0d-8a46-6732b5c3fb25 Info: {"upstream_id":7489990551107061146,"namespace":"default","id":"99ffc8da-5352-4b0d-8a46-6732b5c3fb25","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/?safe-mode=true","create_time":"2025-04-06T08:28:54.885969639+08:00","start_ts":457152747217616897,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457152747217616897,"checkpoint_ts":457152747217616897,"checkpoint_time":"2025-04-06 08:28:54.784"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/kafka_simple_handle_key_only_avro/run.sh using Sink-Type: mysql... <<================= [Sun Apr 6 08:28:55 CST 2025] <<<<<< run test case kafka_simple_handle_key_only_avro success! >>>>>> + set +x =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/kafka_simple_claim_check/run.sh using Sink-Type: mysql... <<================= [Sun Apr 6 08:28:59 CST 2025] <<<<<< run test case kafka_simple_claim_check success! >>>>>> \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } [Pipeline] // timeout [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-periodics_cdc_integration_mysql_test-234/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [2025/04/06 08:29:01.553 +08:00] [INFO] [main.go:78] ["runDDLTest take 1m6.537032997s"] [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } table mark.finish_mark_0 exists table mark.finish_mark_1 exists table mark.finish_mark_2 exists table mark.finish_mark_3 exists table mark.finish_mark_4 exists table mark.finish_mark_5 not exists for 1-th check, retry later table mark.finish_mark_5 not exists for 2-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/kafka_simple_claim_check_avro/run.sh using Sink-Type: mysql... <<================= [Sun Apr 6 08:29:03 CST 2025] <<<<<< run test case kafka_simple_claim_check_avro success! >>>>>> table mark.finish_mark_5 exists table mark.finish_mark not exists for 1-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/canal_json_adapter_compatibility/run.sh using Sink-Type: mysql... <<================= [Sun Apr 6 08:29:07 CST 2025] <<<<<< run test case canal_json_adapter_compatibility success! >>>>>> table mark.finish_mark exists check diff successfully check_redo_resolved_ts 8c6ad029-63cf-4f9d-8684-003eedf81071 457152750787231749 s3://logbucket/test-changefeed?endpoint=http://127.0.0.1:24927/ /tmp/tidb_cdc_test/consistent_replicate_storage_s3/cdc_data/redo/8c6ad029-63cf-4f9d-8684-003eedf81071/meta global resolved ts 457152750498873350 not forward to 457152750787231749 run task failed 1-th time, retry later wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... process cdc.test already exit [Sun Apr 6 08:29:09 CST 2025] <<<<<< run test case multi_source success! >>>>>> check_redo_resolved_ts 8c6ad029-63cf-4f9d-8684-003eedf81071 457152750787231749 s3://logbucket/test-changefeed?endpoint=http://127.0.0.1:24927/ /tmp/tidb_cdc_test/consistent_replicate_storage_s3/cdc_data/redo/8c6ad029-63cf-4f9d-8684-003eedf81071/meta run task successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/canal_json_basic/run.sh using Sink-Type: mysql... <<================= [Sun Apr 6 08:29:12 CST 2025] <<<<<< run test case canal_json_basic success! >>>>>> cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [2025/04/06 08:29:12.768 +08:00] [INFO] [helper.go:54] ["init log"] [file=] [level=info] [2025/04/06 08:29:12.769 +08:00] [INFO] [apply.go:78] ["set memory limit"] [memoryLimit=10737418240] [2025/04/06 08:29:12.769 +08:00] [INFO] [apply.go:94] ["Start http pprof server"] [addr=127.0.0.1:6060] [2025/04/06 08:29:12.770 +08:00] [INFO] [s3.go:423] ["succeed to get bucket region from s3"] ["bucket region"=us-east-1] [2025/04/06 08:29:12.775 +08:00] [INFO] [mem_quota.go:82] ["New memory quota"] [namespace=default] [changefeed=redo-applier] [total=1073741824] [2025/04/06 08:29:12.775 +08:00] [INFO] [redo.go:167] ["apply redo log starts"] [checkpointTs=457152745203302438] [resolvedTs=457152751022899207] [2025/04/06 08:29:12.776 +08:00] [INFO] [s3.go:423] ["succeed to get bucket region from s3"] ["bucket region"=us-east-1] [2025/04/06 08:29:12.776 +08:00] [INFO] [s3.go:423] ["succeed to get bucket region from s3"] ["bucket region"=us-east-1] [2025/04/06 08:29:12.776 +08:00] [INFO] [tz.go:35] ["Use the timezone of the TiCDC server machine"] [timezoneName=System] [timezone=Asia/Shanghai] [2025/04/06 08:29:12.776 +08:00] [WARN] [config.go:428] ["Because time-zone is not specified, the timezone of the TiCDC server will be used. We recommend that you specify the time-zone explicitly. Please make sure that the timezone of the TiCDC server, sink-uri and the downstream database are consistent. If the downstream database does not load the timezone information, you can refer to https://dev.mysql.com/doc/refman/8.0/en/mysql-tzinfo-to-sql.html."] [timezone=Asia/Shanghai] [2025/04/06 08:29:12.776 +08:00] [ERROR] [config.go:216] ["The TiDB source ID should never be set to 0. Please report it as a bug. The default value will be used: 1."] [tidbSourceID=0] [2025/04/06 08:29:12.782 +08:00] [INFO] [file.go:114] ["succeed to download and sort redo logs"] [type=ddl] [duration=6.949779ms] [2025/04/06 08:29:12.783 +08:00] [INFO] [db_helper.go:161] ["sink uri is configured"] [dsn="normal:******@tcp(127.0.0.1:3306)/?interpolateParams=true&multiStatements=true&allow_auto_random_explicit_insert=1&charset=utf8mb4&foreign_key_checks=0&maxAllowedPacket=0&readTimeout=2m&sql_mode=%22IGNORE_SPACE%2CNO_AUTO_VALUE_ON_ZERO%2CALLOW_INVALID_DATES%2CONLY_FULL_GROUP_BY%2CNO_ENGINE_SUBSTITUTION%22&tidb_enable_external_ts_read=%22OFF%22&tidb_placement_mode=%22ignore%22&tidb_txn_mode=optimistic&time_zone=%22Asia%2FShanghai%22&timeout=2m&transaction_isolation=%22READ-COMMITTED%22&writeTimeout=2m"] [2025/04/06 08:29:12.786 +08:00] [INFO] [mysql.go:195] ["MySQL backends is created"] [changefeed=.] [workerCount=16] [forceReplicate=false] [2025/04/06 08:29:12.787 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/06 08:29:12.787 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/06 08:29:12.787 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/06 08:29:12.787 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/06 08:29:12.787 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/06 08:29:12.787 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/06 08:29:12.787 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/06 08:29:12.787 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/06 08:29:12.787 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/06 08:29:12.787 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/06 08:29:12.787 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/06 08:29:12.787 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/06 08:29:12.787 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/06 08:29:12.787 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/06 08:29:12.787 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/06 08:29:12.787 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/06 08:29:12.787 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=3] [2025/04/06 08:29:12.787 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=4] [2025/04/06 08:29:12.787 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=7] [2025/04/06 08:29:12.787 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=10] [2025/04/06 08:29:12.787 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=8] [2025/04/06 08:29:12.787 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=1] [2025/04/06 08:29:12.787 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=0] [2025/04/06 08:29:12.787 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=11] [2025/04/06 08:29:12.787 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=13] [2025/04/06 08:29:12.787 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=14] [2025/04/06 08:29:12.787 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=15] [2025/04/06 08:29:12.787 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=5] [2025/04/06 08:29:12.788 +08:00] [INFO] [tz.go:35] ["Use the timezone of the TiCDC server machine"] [timezoneName=System] [timezone=Asia/Shanghai] [2025/04/06 08:29:12.788 +08:00] [WARN] [config.go:428] ["Because time-zone is not specified, the timezone of the TiCDC server will be used. We recommend that you specify the time-zone explicitly. Please make sure that the timezone of the TiCDC server, sink-uri and the downstream database are consistent. If the downstream database does not load the timezone information, you can refer to https://dev.mysql.com/doc/refman/8.0/en/mysql-tzinfo-to-sql.html."] [timezone=Asia/Shanghai] [2025/04/06 08:29:12.788 +08:00] [ERROR] [config.go:216] ["The TiDB source ID should never be set to 0. Please report it as a bug. The default value will be used: 1."] [tidbSourceID=0] [2025/04/06 08:29:12.787 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=2] [2025/04/06 08:29:12.787 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=12] [2025/04/06 08:29:12.787 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=9] [2025/04/06 08:29:12.787 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=6] [2025/04/06 08:29:12.792 +08:00] [INFO] [db_helper.go:161] ["sink uri is configured"] [dsn="normal:******@tcp(127.0.0.1:3306)/?interpolateParams=true&multiStatements=true&allow_auto_random_explicit_insert=1&charset=utf8mb4&foreign_key_checks=0&maxAllowedPacket=0&readTimeout=2m&sql_mode=%22IGNORE_SPACE%2CNO_AUTO_VALUE_ON_ZERO%2CALLOW_INVALID_DATES%2CONLY_FULL_GROUP_BY%2CNO_ENGINE_SUBSTITUTION%22&tidb_enable_external_ts_read=%22OFF%22&tidb_placement_mode=%22ignore%22&tidb_txn_mode=optimistic&time_zone=%22Asia%2FShanghai%22&timeout=2m&transaction_isolation=%22READ-COMMITTED%22&writeTimeout=2m"] [2025/04/06 08:29:12.797 +08:00] [WARN] [mysql_ddl_sink.go:291] ["please set `has-vector-type` to be true if a column is vector type when the downstream is not TiDB or TiDB version less than specify version"] [hasVectorType=false] [supportVectorVersion=8.4.0] [2025/04/06 08:29:12.797 +08:00] [INFO] [mysql_ddl_sink.go:122] ["MySQL DDL sink is created"] [namespace=] [changefeed=] [2025/04/06 08:29:14.707 +08:00] [INFO] [file.go:114] ["succeed to download and sort redo logs"] [type=row] [duration=1.931677148s] [2025/04/06 08:29:14.708 +08:00] [ERROR] [redo.go:272] ["ignore unsupported DDL"] [ddl="{\"StartTs\":457152745203302433,\"CommitTs\":457152745203302438,\"Query\":\"ALTER TABLE `consistent_replicate_storage_s3`.`t1` EXCHANGE PARTITION `p3` WITH TABLE `consistent_replicate_storage_s3`.`t2`\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"Schema\":\"consistent_replicate_storage_s3\",\"Table\":\"t1\",\"TableID\":118,\"IsPartition\":true},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":42,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false,\"BDRRole\":\"\",\"SQLMode\":0}"] [2025/04/06 08:29:16.167 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [span="{tableID:123, startKey:7480000000000000ff7b5f720000000000fa, endKey:7480000000000000ff7b5f730000000000fa}"] [checkpointTs=457152751022899207] [2025/04/06 08:29:16.167 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [span="{tableID:123, startKey:7480000000000000ff7b5f720000000000fa, endKey:7480000000000000ff7b5f730000000000fa}"] [checkpointTs=457152751022899207] [2025/04/06 08:29:16.268 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [span="{tableID:126, startKey:7480000000000000ff7e5f720000000000fa, endKey:7480000000000000ff7e5f730000000000fa}"] [checkpointTs=457152751022899207] [2025/04/06 08:29:16.268 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [span="{tableID:126, startKey:7480000000000000ff7e5f720000000000fa, endKey:7480000000000000ff7e5f730000000000fa}"] [checkpointTs=457152751022899207] =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/canal_json_content_compatible/run.sh using Sink-Type: mysql... <<================= [Sun Apr 6 08:29:16 CST 2025] <<<<<< run test case canal_json_content_compatible success! >>>>>> [2025/04/06 08:29:16.770 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [span="{tableID:128, startKey:7480000000000000ff805f720000000000fa, endKey:7480000000000000ff805f730000000000fa}"] [checkpointTs=457152751022899207] [2025/04/06 08:29:16.770 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [span="{tableID:128, startKey:7480000000000000ff805f720000000000fa, endKey:7480000000000000ff805f730000000000fa}"] [checkpointTs=457152751022899207] [2025/04/06 08:29:16.870 +08:00] [INFO] [redo.go:223] ["apply redo log finishes"] [appliedLogCount=5008] [appliedDDLCount=0] [currentCheckpoint=457152751022899207] [2025/04/06 08:29:16.870 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=0] [2025/04/06 08:29:16.870 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=6] [2025/04/06 08:29:16.870 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=12] [2025/04/06 08:29:16.871 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=11] [2025/04/06 08:29:16.871 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=13] [2025/04/06 08:29:16.870 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=2] [2025/04/06 08:29:16.871 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=1] [2025/04/06 08:29:16.871 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=14] [2025/04/06 08:29:16.871 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=5] [2025/04/06 08:29:16.871 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=10] [2025/04/06 08:29:16.870 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=15] [2025/04/06 08:29:16.871 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=3] [2025/04/06 08:29:16.870 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=4] [2025/04/06 08:29:16.871 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=9] [2025/04/06 08:29:16.871 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=8] [2025/04/06 08:29:16.871 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=7] Apply redo log successfully check diff successfully [Sun Apr 6 08:29:17 CST 2025] <<<<<< run test case consistent_replicate_storage_s3 success! >>>>>> count(*) 5000 Exiting on signal: INTERRUPT =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/multi_topics/run.sh using Sink-Type: mysql... <<================= [Sun Apr 6 08:29:20 CST 2025] <<<<<< run test case multi_topics success! >>>>>> ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 221 100 221 0 0 2750 0 --:--:-- --:--:-- --:--:-- 2762 + synced_status='{"synced":true,"sink_checkpoint_ts":"2025-04-06 08:29:10.334","puller_resolved_ts":"2025-04-06 08:29:02.334","last_synced_ts":"2025-04-06 08:26:53.435","now_ts":"2025-04-06 08:29:10.000","info":"Data syncing is finished"}' ++ echo '{"synced":true,"sink_checkpoint_ts":"2025-04-06' '08:29:10.334","puller_resolved_ts":"2025-04-06' '08:29:02.334","last_synced_ts":"2025-04-06' '08:26:53.435","now_ts":"2025-04-06' '08:29:10.000","info":"Data' syncing is 'finished"}' ++ jq .synced + status=true + '[' true '!=' true ']' + kill_pd ++ ps aux ++ grep pd-server ++ grep /tmp/tidb_cdc_test/synced_status + info='jenkins 15321 9.2 0.0 14380708 125568 ? Sl 08:26 0:14 pd-server --advertise-client-urls http://127.0.0.1:2379 --client-urls http://0.0.0.0:2379 --advertise-peer-urls http://127.0.0.1:2380 --peer-urls http://0.0.0.0:2380 --config /tmp/tidb_cdc_test/synced_status/pd-config.toml --log-file /tmp/tidb_cdc_test/synced_status/pd1.log --data-dir /tmp/tidb_cdc_test/synced_status/pd1 --name=pd1 --initial-cluster=pd1=http://127.0.0.1:2380 jenkins 15393 6.7 0.0 14516648 124416 ? Sl 08:26 0:10 pd-server --advertise-client-urls http://127.0.0.1:2479 --client-urls http://0.0.0.0:2479 --advertise-peer-urls http://127.0.0.1:2480 --peer-urls http://0.0.0.0:2480 --config /tmp/tidb_cdc_test/synced_status/pd-config.toml --log-file /tmp/tidb_cdc_test/synced_status/down_pd.log --data-dir /tmp/tidb_cdc_test/synced_status/down_pd' ++ ps aux ++ grep pd-server ++ grep /tmp/tidb_cdc_test/synced_status ++ awk '{print $2}' ++ xargs kill -9 + sleep 20 {"level":"warn","ts":1743899357.7382047,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc003ffefc0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":1743899357.7382858,"caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":1743899357.7417574,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc002e1e700/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":1743899357.7418215,"caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":1743899357.780642,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0049d9880/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} {"level":"info","ts":1743899357.7807047,"caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/vector/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/avro_basic/run.sh using Sink-Type: mysql... <<================= [Sun Apr 6 08:29:24 CST 2025] <<<<<< run test case avro_basic success! >>>>>> {"level":"warn","ts":"2025-04-06T08:29:25.177581+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001e24e00/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2025-04-06T08:29:25.178072+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001f12380/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2025-04-06T08:29:25.283438+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001e80e00/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} start tidb cluster in /tmp/tidb_cdc_test/vector Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/consistent_partition_table/run.sh using Sink-Type: mysql... <<================= The 1 times to try to start tidb cluster... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/canal_json_handle_key_only/run.sh using Sink-Type: mysql... <<================= [Sun Apr 6 08:29:28 CST 2025] <<<<<< run test case canal_json_handle_key_only success! >>>>>> Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/consistent_partition_table Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0{"level":"warn","ts":1743899370.8280323,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc003ffefc0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":1743899370.8689926,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0049d9880/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} {"level":"warn","ts":"2025-04-06T08:29:31.17839+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001e24e00/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2025-04-06T08:29:31.17924+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001f12380/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2025-04-06T08:29:31.285097+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001e80e00/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release 0 0 0 0 0 0 0 0 --:--:-- 0:00:01 --:--:-- 0 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/open_protocol_handle_key_only/run.sh using Sink-Type: mysql... <<================= [Sun Apr 6 08:29:32 CST 2025] <<<<<< run test case open_protocol_handle_key_only success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) 0 0 0 0 0 0 0 0 --:--:-- 0:00:02 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:03 --:--:-- 0 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489990751847537716 TiDB Cluster ID. tikv_gc_leader_uuid 6582202bc9c0006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-v-hm9ds, pid:29891, start at 2025-04-06 08:29:33.35976012 +0800 CST m=+2.506360323 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:31:33.368 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:29:33.351 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:19:33.351 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489990751847537716 TiDB Cluster ID. tikv_gc_leader_uuid 6582202bc9c0006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-v-hm9ds, pid:29891, start at 2025-04-06 08:29:33.35976012 +0800 CST m=+2.506360323 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:31:33.368 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:29:33.351 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:19:33.351 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489990755690154376 TiDB Cluster ID. tikv_gc_leader_uuid 6582202bf540009 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-v-hm9ds, pid:29966, start at 2025-04-06 08:29:33.542041411 +0800 CST m=+2.610028227 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:31:33.551 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:29:33.555 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:19:33.555 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/vector/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/vector/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/vector/tiflash/db/proxy"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/vector/tiflash/log/proxy.log"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/vector/tiflash-proxy.toml"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) 0 0 0 0 0 0 0 0 --:--:-- 0:00:04 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:05 --:--:-- 0 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/canal_json_claim_check/run.sh using Sink-Type: mysql... <<================= [Sun Apr 6 08:29:37 CST 2025] <<<<<< run test case canal_json_claim_check success! >>>>>> 0 0 0 0 0 0 0 0 --:--:-- 0:00:06 --:--:-- 0{"level":"warn","ts":"2025-04-06T08:29:37.180474+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001f12380/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2025-04-06T08:29:37.180508+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001e24e00/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2025-04-06T08:29:37.28651+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001e80e00/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.vector.cli.31223.out cli tso query --pd=http://127.0.0.1:2379 0 0 0 0 0 0 0 0 --:--:-- 0:00:07 --:--:-- 0 + set +x + tso='457152758454943745 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 457152758454943745 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Sun Apr 6 08:29:39 CST 2025] <<<<<< START cdc server in vector case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.vector.3125931261.out server --log-file /tmp/tidb_cdc_test/vector/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/vector/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489990768030932754 TiDB Cluster ID. tikv_gc_leader_uuid 6582202fe30001d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-6-zhp59, pid:19014, start at 2025-04-06 08:29:37.589669213 +0800 CST m=+2.536026947 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:31:37.642 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:29:37.629 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:19:37.629 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489990768030932754 TiDB Cluster ID. tikv_gc_leader_uuid 6582202fe30001d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-6-zhp59, pid:19014, start at 2025-04-06 08:29:37.589669213 +0800 CST m=+2.536026947 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:31:37.642 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:29:37.629 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:19:37.629 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489990769258579042 TiDB Cluster ID. tikv_gc_leader_uuid 6582202fe28000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-6-zhp59, pid:19093, start at 2025-04-06 08:29:37.573122005 +0800 CST m=+2.434369853 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:31:37.582 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:29:37.596 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:19:37.596 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/consistent_partition_table/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/consistent_partition_table/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/consistent_partition_table/tiflash/db/proxy"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/consistent_partition_table/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/consistent_partition_table/tiflash-proxy.toml"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } 0 0 0 0 0 0 0 0 --:--:-- 0:00:08 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:09 --:--:-- 0 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/open_protocol_claim_check/run.sh using Sink-Type: mysql... <<================= [Sun Apr 6 08:29:41 CST 2025] <<<<<< run test case open_protocol_claim_check success! >>>>>> [Sun Apr 6 08:29:41 CST 2025] <<<<<< START cdc server in consistent_partition_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.consistent_partition_table.2035020352.out server --log-file /tmp/tidb_cdc_test/consistent_partition_table/cdcpartition_table.server1.log --log-level debug --data-dir /tmp/tidb_cdc_test/consistent_partition_table/cdc_datapartition_table.server1 --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 0 0 0 0 0 0 0 0 --:--:-- 0:00:10 --:--:-- 0 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 06 Apr 2025 00:29:42 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ee581946-48c4-4861-8c2e-7d82605e4503 {"id":"ee581946-48c4-4861-8c2e-7d82605e4503","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899379} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319608808f7ee5 ee581946-48c4-4861-8c2e-7d82605e4503 /tidb/cdc/default/default/upstream/7489990751847537716 {"id":7489990751847537716,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ee581946-48c4-4861-8c2e-7d82605e4503 {"id":"ee581946-48c4-4861-8c2e-7d82605e4503","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899379} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319608808f7ee5 ee581946-48c4-4861-8c2e-7d82605e4503 /tidb/cdc/default/default/upstream/7489990751847537716 {"id":7489990751847537716,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ee581946-48c4-4861-8c2e-7d82605e4503 {"id":"ee581946-48c4-4861-8c2e-7d82605e4503","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899379} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319608808f7ee5 ee581946-48c4-4861-8c2e-7d82605e4503 /tidb/cdc/default/default/upstream/7489990751847537716 {"id":7489990751847537716,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.vector.cli.31317.out cli changefeed create --start-ts=457152758454943745 --sink-uri=mysql://normal:123456@127.0.0.1:3306/ Create changefeed successfully! ID: 9606e709-0524-481f-a081-576b70615f0a Info: {"upstream_id":7489990751847537716,"namespace":"default","id":"9606e709-0524-481f-a081-576b70615f0a","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-06T08:29:42.599946933+08:00","start_ts":457152758454943745,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457152758454943745,"checkpoint_ts":457152758454943745,"checkpoint_time":"2025-04-06 08:29:37.651"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... table safe_mode.finish_mark exists check diff successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Sun Apr 6 08:29:38 CST 2025] <<<<<< run test case safe_mode success! >>>>>> 0 0 0 0 0 0 0 0 --:--:-- 0:00:11 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:12 --:--:-- 0{"level":"warn","ts":"2025-04-06T08:29:43.181421+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001e24e00/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2025-04-06T08:29:43.182066+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001f12380/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2025-04-06T08:29:43.2881+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001e80e00/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 06 Apr 2025 00:29:44 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/39d0f751-6f87-4c0a-8586-29f88a106b2c {"id":"39d0f751-6f87-4c0a-8586-29f88a106b2c","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899381} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/2231960880a5bed6 39d0f751-6f87-4c0a-8586-29f88a106b2c /tidb/cdc/default/default/upstream/7489990768030932754 {"id":7489990768030932754,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/39d0f751-6f87-4c0a-8586-29f88a106b2c {"id":"39d0f751-6f87-4c0a-8586-29f88a106b2c","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899381} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/2231960880a5bed6 39d0f751-6f87-4c0a-8586-29f88a106b2c /tidb/cdc/default/default/upstream/7489990768030932754 {"id":7489990768030932754,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/39d0f751-6f87-4c0a-8586-29f88a106b2c {"id":"39d0f751-6f87-4c0a-8586-29f88a106b2c","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899381} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/2231960880a5bed6 39d0f751-6f87-4c0a-8586-29f88a106b2c /tidb/cdc/default/default/upstream/7489990768030932754 {"id":7489990768030932754,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + set +x 0 0 0 0 0 0 0 0 --:--:-- 0:00:13 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:14 --:--:-- 0{"level":"warn","ts":"2025-04-06T08:29:45.166071+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001e24e00/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":"2025-04-06T08:29:45.166134+0800","logger":"etcd-client","caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":"2025-04-06T08:29:45.167098+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001f12380/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":"2025-04-06T08:29:45.167139+0800","logger":"etcd-client","caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":"2025-04-06T08:29:45.272496+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001e80e00/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} {"level":"info","ts":"2025-04-06T08:29:45.272534+0800","logger":"etcd-client","caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/canal_json_storage_basic/run.sh using Sink-Type: mysql... <<================= [Sun Apr 6 08:29:45 CST 2025] <<<<<< run test case canal_json_storage_basic success! >>>>>> 0 0 0 0 0 0 0 0 --:--:-- 0:00:15 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:16 --:--:-- 0 table test.finish_mark not exists for 1-th check, retry later table partition_table2.t2 not exists for 1-th check, retry later 0 0 0 0 0 0 0 0 --:--:-- 0:00:17 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:18 --:--:-- 0{"level":"warn","ts":"2025-04-06T08:29:49.18251+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001e24e00/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2025-04-06T08:29:49.183334+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001f12380/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2025-04-06T08:29:49.289835+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001e80e00/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/kafka_big_messages/run.sh using Sink-Type: mysql... <<================= [Sun Apr 6 08:29:49 CST 2025] <<<<<< run test case kafka_big_messages success! >>>>>> table test.finish_mark not exists for 2-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/canal_json_storage_partition_table/run.sh using Sink-Type: mysql... <<================= [Sun Apr 6 08:29:49 CST 2025] <<<<<< run test case canal_json_storage_partition_table success! >>>>>> table partition_table2.t2 not exists for 2-th check, retry later 0 0 0 0 0 0 0 0 --:--:-- 0:00:19 --:--:-- 0 table test.finish_mark exists check diff successfully 0 0 0 0 0 0 0 0 --:--:-- 0:00:20 --:--:-- 0 wait process cdc.test exit for 1-th time... table partition_table2.t2 not exists for 3-th check, retry later wait process cdc.test exit for 2-th time... 0 0 0 0 0 0 0 0 --:--:-- 0:00:21 --:--:-- 0{"level":"warn","ts":1743899392.7403789,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc003ffefc0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":1743899392.7404265,"caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":1743899392.7437153,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc002e1e700/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":1743899392.7437673,"caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} 0 0 0 0 0 0 0 0 --:--:-- 0:00:22 --:--:-- 0{"level":"warn","ts":1743899392.7821047,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0049d9880/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} {"level":"info","ts":1743899392.7821407,"caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Sun Apr 6 08:29:53 CST 2025] <<<<<< run test case vector success! >>>>>> =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/kafka_compression/run.sh using Sink-Type: mysql... <<================= [Sun Apr 6 08:29:53 CST 2025] <<<<<< run test case kafka_compression success! >>>>>> =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/multi_tables_ddl/run.sh using Sink-Type: mysql... <<================= * About to connect() to 127.0.0.1 port 24927 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:24927; Connection refused * Closing connection 0 You are running an older version of MinIO released 4 years ago Update: Run `mc admin update` Attempting encryption of all config, IAM users and policies on MinIO backend table partition_table2.t2 not exists for 4-th check, retry later Endpoint: http://127.0.0.1:24927 Object API (Amazon S3 compatible): Go: https://docs.min.io/docs/golang-client-quickstart-guide Java: https://docs.min.io/docs/java-client-quickstart-guide Python: https://docs.min.io/docs/python-client-quickstart-guide JavaScript: https://docs.min.io/docs/javascript-client-quickstart-guide .NET: https://docs.min.io/docs/dotnet-client-quickstart-guide 0 0 0 0 0 0 0 0 --:--:-- 0:00:23 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:24 --:--:-- 0{"level":"warn","ts":"2025-04-06T08:29:55.184488+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001f12380/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2025-04-06T08:29:55.184608+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001e24e00/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2025-04-06T08:29:55.291433+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001e80e00/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} * About to connect() to 127.0.0.1 port 24927 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 24927 (#0) > GET / HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:24927 > Accept: */* > < HTTP/1.1 403 Forbidden < Accept-Ranges: bytes < Content-Length: 226 < Content-Security-Policy: block-all-mixed-content < Content-Type: application/xml < Server: MinIO/RELEASE.2020-07-27T18-37-02Z < Vary: Origin < X-Amz-Request-Id: 18339342BF6D4AD5 < X-Xss-Protection: 1; mode=block < Date: Sun, 06 Apr 2025 00:29:55 GMT < { [data not shown] * Connection #0 to host 127.0.0.1 left intact Bucket 's3://logbucket/' created The 1 times to try to start tidb cluster... table partition_table2.t2 not exists for 5-th check, retry later 0 0 0 0 0 0 0 0 --:--:-- 0:00:25 --:--:-- 0 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/kafka_messages/run.sh using Sink-Type: mysql... <<================= [Sun Apr 6 08:29:57 CST 2025] <<<<<< run test case kafka_messages success! >>>>>> 0 0 0 0 0 0 0 0 --:--:-- 0:00:26 --:--:-- 0 table partition_table2.t2 not exists for 6-th check, retry later 0 0 0 0 0 0 0 0 --:--:-- 0:00:27 --:--:-- 0 start tidb cluster in /tmp/tidb_cdc_test/multi_tables_ddl Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... 0 0 0 0 0 0 0 0 --:--:-- 0:00:28 --:--:-- 0 table partition_table2.t2 not exists for 7-th check, retry later 0 0 0 0 0 0 0 0 --:--:-- 0:00:29 --:--:-- 0 100 135 100 135 0 0 4 0 0:00:33 0:00:30 0:00:03 27 100 135 100 135 0 0 4 0 0:00:33 0:00:30 0:00:03 33 + synced_status='{ "error_msg": "[CDC:ErrPDEtcdAPIError]etcd api call error: context deadline exceeded", "error_code": "CDC:ErrPDEtcdAPIError" }' ++ jq -r .error_code ++ echo '{' '"error_msg":' '"[CDC:ErrPDEtcdAPIError]etcd' api call error: context deadline 'exceeded",' '"error_code":' '"CDC:ErrPDEtcdAPIError"' '}' + error_code=CDC:ErrPDEtcdAPIError + cleanup_process cdc.test {"level":"warn","ts":"2025-04-06T08:30:01.185361+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001e24e00/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2025-04-06T08:30:01.185712+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001f12380/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2025-04-06T08:30:01.292751+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001e80e00/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} wait process cdc.test exit for 1-th time... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/kafka_sink_error_resume/run.sh using Sink-Type: mysql... <<================= [Sun Apr 6 08:30:01 CST 2025] <<<<<< run test case kafka_sink_error_resume success! >>>>>> wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit + stop_tidb_cluster table partition_table2.t2 exists Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... \033[0;36m<<< Run all test success >>>\033[0m Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] } [Pipeline] // timeout [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-periodics_cdc_integration_mysql_test-234/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/mq_sink_lost_callback/run.sh using Sink-Type: mysql... <<================= [Sun Apr 6 08:30:06 CST 2025] <<<<<< run test case mq_sink_lost_callback success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Sun Apr 6 08:30:08 CST 2025] <<<<<< START cdc server in consistent_partition_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sink/dmlsink/txn/mysql/MySQLSinkHangLongTime=return(true);github.com/pingcap/tiflow/cdc/sink/ddlsink/mysql/MySQLSinkExecDDLDelay=return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.consistent_partition_table.2053920541.out server --log-file /tmp/tidb_cdc_test/consistent_partition_table/cdcpartition_table.server2.log --log-level debug --data-dir /tmp/tidb_cdc_test/consistent_partition_table/cdc_datapartition_table.server2 --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489990900010735562 TiDB Cluster ID. tikv_gc_leader_uuid 6582204ccc80013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-8-31x9h, pid:47334, start at 2025-04-06 08:30:07.172750494 +0800 CST m=+2.026737533 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:32:07.180 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:30:07.223 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:20:07.223 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489990900010735562 TiDB Cluster ID. tikv_gc_leader_uuid 6582204ccc80013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-8-31x9h, pid:47334, start at 2025-04-06 08:30:07.172750494 +0800 CST m=+2.026737533 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:32:07.180 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:30:07.223 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:20:07.223 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489990902763916524 TiDB Cluster ID. tikv_gc_leader_uuid 6582204cddc000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-8-31x9h, pid:47411, start at 2025-04-06 08:30:07.243341495 +0800 CST m=+2.016958266 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:32:07.250 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:30:07.251 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:20:07.251 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/multi_tables_ddl/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/multi_tables_ddl/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/multi_tables_ddl/tiflash/db/proxy"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/multi_tables_ddl/tiflash/log/proxy.log"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/multi_tables_ddl/tiflash-proxy.toml"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/mq_sink_dispatcher/run.sh using Sink-Type: mysql... <<================= [Sun Apr 6 08:30:10 CST 2025] <<<<<< run test case mq_sink_dispatcher success! >>>>>> [Sun Apr 6 08:30:11 CST 2025] <<<<<< START cdc server in multi_tables_ddl case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_tables_ddl.4870748709.out server --log-file /tmp/tidb_cdc_test/multi_tables_ddl/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_tables_ddl/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 06 Apr 2025 00:30:11 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/aea60c9d-25a3-44b9-b88e-98c6aa94270b {UpstreamID:7489990768030932754 Namespace:default ID:aea60c9d-25a3-44b9-b88e-98c6aa94270b SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-06 08:29:44.6932829 +0800 CST StartTs:457152760276320263 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0037cf680 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457152760289427467} {CheckpointTs:457152765925523461 MinTableBarrierTs:457152767235981317 AdminJobType:noop} span: {tableID:124, startKey:7480000000000000ff7c5f720000000000fa, endKey:7480000000000000ff7c5f730000000000fa}, resolvedTs: 457152765925523461, checkpointTs: 457152765925523461, state: Replicating span: {tableID:119, startKey:7480000000000000ff775f720000000000fa, endKey:7480000000000000ff775f730000000000fa}, resolvedTs: 457152765925523461, checkpointTs: 457152765925523461, state: Replicating span: {tableID:122, startKey:7480000000000000ff7a5f720000000000fa, endKey:7480000000000000ff7a5f730000000000fa}, resolvedTs: 457152765925523461, checkpointTs: 457152765925523461, state: Replicating span: {tableID:127, startKey:7480000000000000ff7f5f720000000000fa, endKey:7480000000000000ff7f5f730000000000fa}, resolvedTs: 457152765925523461, checkpointTs: 457152765925523461, state: Replicating span: {tableID:123, startKey:7480000000000000ff7b5f720000000000fa, endKey:7480000000000000ff7b5f730000000000fa}, resolvedTs: 457152765925523461, checkpointTs: 457152765925523461, state: Replicating span: {tableID:129, startKey:7480000000000000ff815f720000000000fa, endKey:7480000000000000ff815f730000000000fa}, resolvedTs: 457152765925523461, checkpointTs: 457152765925523461, state: Replicating span: {tableID:117, startKey:7480000000000000ff755f720000000000fa, endKey:7480000000000000ff755f730000000000fa}, resolvedTs: 457152765925523461, checkpointTs: 457152765925523461, state: Replicating span: {tableID:126, startKey:7480000000000000ff7e5f720000000000fa, endKey:7480000000000000ff7e5f730000000000fa}, resolvedTs: 457152765925523461, checkpointTs: 457152765925523461, state: Replicating span: {tableID:118, startKey:7480000000000000ff765f720000000000fa, endKey:7480000000000000ff765f730000000000fa}, resolvedTs: 457152765925523461, checkpointTs: 457152765925523461, state: Replicating span: {tableID:116, startKey:7480000000000000ff745f720000000000fa, endKey:7480000000000000ff745f730000000000fa}, resolvedTs: 457152765925523461, checkpointTs: 457152765925523461, state: Replicating span: {tableID:115, startKey:7480000000000000ff735f720000000000fa, endKey:7480000000000000ff735f730000000000fa}, resolvedTs: 457152765925523461, checkpointTs: 457152765925523461, state: Replicating span: {tableID:133, startKey:7480000000000000ff855f720000000000fa, endKey:7480000000000000ff855f730000000000fa}, resolvedTs: 457152765925523461, checkpointTs: 457152765925523461, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/55fd4491-14e9-497b-ac92-bb82c88c9813 {"id":"55fd4491-14e9-497b-ac92-bb82c88c9813","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899409} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/2231960880a5c02e 55fd4491-14e9-497b-ac92-bb82c88c9813 /tidb/cdc/default/default/changefeed/info/aea60c9d-25a3-44b9-b88e-98c6aa94270b {"upstream-id":7489990768030932754,"namespace":"default","changefeed-id":"aea60c9d-25a3-44b9-b88e-98c6aa94270b","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-06T08:29:44.6932829+08:00","start-ts":457152760276320263,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"file:///tmp/tidb_cdc_test/consistent_partition_table/redo","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457152760289427467} /tidb/cdc/default/default/changefeed/status/aea60c9d-25a3-44b9-b88e-98c6aa94270b {"checkpoint-ts":457152765925523461,"min-table-barrier-ts":457152767235981317,"admin-job-type":0} /tidb/cdc/default/default/task/position/55fd4491-14e9-497b-ac92-bb82c88c9813/aea60c9d-25a3-44b9-b88e-98c6aa94270b {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489990768030932754 {"id":7489990768030932754,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/aea60c9d-25a3-44b9-b88e-98c6aa94270b {UpstreamID:7489990768030932754 Namespace:default ID:aea60c9d-25a3-44b9-b88e-98c6aa94270b SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-06 08:29:44.6932829 +0800 CST StartTs:457152760276320263 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0037cf680 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457152760289427467} {CheckpointTs:457152765925523461 MinTableBarrierTs:457152767235981317 AdminJobType:noop} span: {tableID:124, startKey:7480000000000000ff7c5f720000000000fa, endKey:7480000000000000ff7c5f730000000000fa}, resolvedTs: 457152765925523461, checkpointTs: 457152765925523461, state: Replicating span: {tableID:119, startKey:7480000000000000ff775f720000000000fa, endKey:7480000000000000ff775f730000000000fa}, resolvedTs: 457152765925523461, checkpointTs: 457152765925523461, state: Replicating span: {tableID:122, startKey:7480000000000000ff7a5f720000000000fa, endKey:7480000000000000ff7a5f730000000000fa}, resolvedTs: 457152765925523461, checkpointTs: 457152765925523461, state: Replicating span: {tableID:127, startKey:7480000000000000ff7f5f720000000000fa, endKey:7480000000000000ff7f5f730000000000fa}, resolvedTs: 457152765925523461, checkpointTs: 457152765925523461, state: Replicating span: {tableID:123, startKey:7480000000000000ff7b5f720000000000fa, endKey:7480000000000000ff7b5f730000000000fa}, resolvedTs: 457152765925523461, checkpointTs: 457152765925523461, state: Replicating span: {tableID:129, startKey:7480000000000000ff815f720000000000fa, endKey:7480000000000000ff815f730000000000fa}, resolvedTs: 457152765925523461, checkpointTs: 457152765925523461, state: Replicating span: {tableID:117, startKey:7480000000000000ff755f720000000000fa, endKey:7480000000000000ff755f730000000000fa}, resolvedTs: 457152765925523461, checkpointTs: 457152765925523461, state: Replicating span: {tableID:126, startKey:7480000000000000ff7e5f720000000000fa, endKey:7480000000000000ff7e5f730000000000fa}, resolvedTs: 457152765925523461, checkpointTs: 457152765925523461, state: Replicating span: {tableID:118, startKey:7480000000000000ff765f720000000000fa, endKey:7480000000000000ff765f730000000000fa}, resolvedTs: 457152765925523461, checkpointTs: 457152765925523461, state: Replicating span: {tableID:116, startKey:7480000000000000ff745f720000000000fa, endKey:7480000000000000ff745f730000000000fa}, resolvedTs: 457152765925523461, checkpointTs: 457152765925523461, state: Replicating span: {tableID:115, startKey:7480000000000000ff735f720000000000fa, endKey:7480000000000000ff735f730000000000fa}, resolvedTs: 457152765925523461, checkpointTs: 457152765925523461, state: Replicating span: {tableID:133, startKey:7480000000000000ff855f720000000000fa, endKey:7480000000000000ff855f730000000000fa}, resolvedTs: 457152765925523461, checkpointTs: 457152765925523461, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/55fd4491-14e9-497b-ac92-bb82c88c9813 {"id":"55fd4491-14e9-497b-ac92-bb82c88c9813","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899409} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/2231960880a5c02e 55fd4491-14e9-497b-ac92-bb82c88c9813 /tidb/cdc/default/default/changefeed/info/aea60c9d-25a3-44b9-b88e-98c6aa94270b {"upstream-id":7489990768030932754,"namespace":"default","changefeed-id":"aea60c9d-25a3-44b9-b88e-98c6aa94270b","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-06T08:29:44.6932829+08:00","start-ts":457152760276320263,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"file:///tmp/tidb_cdc_test/consistent_partition_table/redo","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457152760289427467} /tidb/cdc/default/default/changefeed/status/aea60c9d-25a3-44b9-b88e-98c6aa94270b {"checkpoint-ts":457152765925523461,"min-table-barrier-ts":457152767235981317,"admin-job-type":0} /tidb/cdc/default/default/task/position/55fd4491-14e9-497b-ac92-bb82c88c9813/aea60c9d-25a3-44b9-b88e-98c6aa94270b {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489990768030932754 {"id":7489990768030932754,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + grep -q 'etcd info' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/aea60c9d-25a3-44b9-b88e-98c6aa94270b {UpstreamID:7489990768030932754 Namespace:default ID:aea60c9d-25a3-44b9-b88e-98c6aa94270b SinkURI:mysql://normal:123456@127.0.0.1:3306/ CreateTime:2025-04-06 08:29:44.6932829 +0800 CST StartTs:457152760276320263 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0037cf680 State:normal Error: Warning: CreatorVersion:v9.0.0-beta.1-9-g4c631d595 Epoch:457152760289427467} {CheckpointTs:457152765925523461 MinTableBarrierTs:457152767235981317 AdminJobType:noop} span: {tableID:124, startKey:7480000000000000ff7c5f720000000000fa, endKey:7480000000000000ff7c5f730000000000fa}, resolvedTs: 457152765925523461, checkpointTs: 457152765925523461, state: Replicating span: {tableID:119, startKey:7480000000000000ff775f720000000000fa, endKey:7480000000000000ff775f730000000000fa}, resolvedTs: 457152765925523461, checkpointTs: 457152765925523461, state: Replicating span: {tableID:122, startKey:7480000000000000ff7a5f720000000000fa, endKey:7480000000000000ff7a5f730000000000fa}, resolvedTs: 457152765925523461, checkpointTs: 457152765925523461, state: Replicating span: {tableID:127, startKey:7480000000000000ff7f5f720000000000fa, endKey:7480000000000000ff7f5f730000000000fa}, resolvedTs: 457152765925523461, checkpointTs: 457152765925523461, state: Replicating span: {tableID:123, startKey:7480000000000000ff7b5f720000000000fa, endKey:7480000000000000ff7b5f730000000000fa}, resolvedTs: 457152765925523461, checkpointTs: 457152765925523461, state: Replicating span: {tableID:129, startKey:7480000000000000ff815f720000000000fa, endKey:7480000000000000ff815f730000000000fa}, resolvedTs: 457152765925523461, checkpointTs: 457152765925523461, state: Replicating span: {tableID:117, startKey:7480000000000000ff755f720000000000fa, endKey:7480000000000000ff755f730000000000fa}, resolvedTs: 457152765925523461, checkpointTs: 457152765925523461, state: Replicating span: {tableID:126, startKey:7480000000000000ff7e5f720000000000fa, endKey:7480000000000000ff7e5f730000000000fa}, resolvedTs: 457152765925523461, checkpointTs: 457152765925523461, state: Replicating span: {tableID:118, startKey:7480000000000000ff765f720000000000fa, endKey:7480000000000000ff765f730000000000fa}, resolvedTs: 457152765925523461, checkpointTs: 457152765925523461, state: Replicating span: {tableID:116, startKey:7480000000000000ff745f720000000000fa, endKey:7480000000000000ff745f730000000000fa}, resolvedTs: 457152765925523461, checkpointTs: 457152765925523461, state: Replicating span: {tableID:115, startKey:7480000000000000ff735f720000000000fa, endKey:7480000000000000ff735f730000000000fa}, resolvedTs: 457152765925523461, checkpointTs: 457152765925523461, state: Replicating span: {tableID:133, startKey:7480000000000000ff855f720000000000fa, endKey:7480000000000000ff855f730000000000fa}, resolvedTs: 457152765925523461, checkpointTs: 457152765925523461, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/55fd4491-14e9-497b-ac92-bb82c88c9813 {"id":"55fd4491-14e9-497b-ac92-bb82c88c9813","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899409} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/2231960880a5c02e 55fd4491-14e9-497b-ac92-bb82c88c9813 /tidb/cdc/default/default/changefeed/info/aea60c9d-25a3-44b9-b88e-98c6aa94270b {"upstream-id":7489990768030932754,"namespace":"default","changefeed-id":"aea60c9d-25a3-44b9-b88e-98c6aa94270b","sink-uri":"mysql://normal:123456@127.0.0.1:3306/","create-time":"2025-04-06T08:29:44.6932829+08:00","start-ts":457152760276320263,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"send-all-bootstrap-at-start":false,"debezium-disable-schema":false,"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"eventual","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"file:///tmp/tidb_cdc_test/consistent_partition_table/redo","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v9.0.0-beta.1-9-g4c631d595","epoch":457152760289427467} /tidb/cdc/default/default/changefeed/status/aea60c9d-25a3-44b9-b88e-98c6aa94270b {"checkpoint-ts":457152765925523461,"min-table-barrier-ts":457152767235981317,"admin-job-type":0} /tidb/cdc/default/default/task/position/55fd4491-14e9-497b-ac92-bb82c88c9813/aea60c9d-25a3-44b9-b88e-98c6aa94270b {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7489990768030932754 {"id":7489990768030932754,"pd-endpoints":"","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x + run_case_with_unavailable_tikv conf/changefeed.toml + rm -rf /tmp/tidb_cdc_test/synced_status + mkdir -p /tmp/tidb_cdc_test/synced_status + start_tidb_cluster --workdir /tmp/tidb_cdc_test/synced_status shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory The 1 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 06 Apr 2025 00:30:14 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/bdbe02ce-e540-4afc-8e8d-c636f178c248 {"id":"bdbe02ce-e540-4afc-8e8d-c636f178c248","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899411} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319608811770d5 bdbe02ce-e540-4afc-8e8d-c636f178c248 /tidb/cdc/default/default/upstream/7489990900010735562 {"id":7489990900010735562,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/bdbe02ce-e540-4afc-8e8d-c636f178c248 {"id":"bdbe02ce-e540-4afc-8e8d-c636f178c248","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899411} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319608811770d5 bdbe02ce-e540-4afc-8e8d-c636f178c248 /tidb/cdc/default/default/upstream/7489990900010735562 {"id":7489990900010735562,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/bdbe02ce-e540-4afc-8e8d-c636f178c248 {"id":"bdbe02ce-e540-4afc-8e8d-c636f178c248","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899411} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319608811770d5 bdbe02ce-e540-4afc-8e8d-c636f178c248 /tidb/cdc/default/default/upstream/7489990900010735562 {"id":7489990900010735562,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: test-normal Info: {"upstream_id":7489990900010735562,"namespace":"default","id":"test-normal","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-06T08:30:14.815012918+08:00","start_ts":457152767342411777,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["multi_tables_ddl_test.t1","multi_tables_ddl_test.t2","multi_tables_ddl_test.t3","multi_tables_ddl_test.t4","multi_tables_ddl_test.t1_7","multi_tables_ddl_test.t2_7","multi_tables_ddl_test.finish_mark"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":true,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457152767342411777,"checkpoint_ts":457152767342411777,"checkpoint_time":"2025-04-06 08:30:11.554"} Create changefeed successfully! ID: test-error-1 Info: {"upstream_id":7489990900010735562,"namespace":"default","id":"test-error-1","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-06T08:30:14.948353638+08:00","start_ts":457152767342411777,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["multi_tables_ddl_test.t5","multi_tables_ddl_test.t6","multi_tables_ddl_test.t7","multi_tables_ddl_test.t8"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":true,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457152767342411777,"checkpoint_ts":457152767342411777,"checkpoint_time":"2025-04-06 08:30:11.554"} Create changefeed successfully! ID: test-error-2 Info: {"upstream_id":7489990900010735562,"namespace":"default","id":"test-error-2","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2025-04-06T08:30:15.114445846+08:00","start_ts":457152767342411777,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["multi_tables_ddl_test.t9","multi_tables_ddl_test.t10"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":true,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457152767342411777,"checkpoint_ts":457152767342411777,"checkpoint_time":"2025-04-06 08:30:11.554"} =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/kafka_column_selector/run.sh using Sink-Type: mysql... <<================= [Sun Apr 6 08:30:14 CST 2025] <<<<<< run test case kafka_column_selector success! >>>>>> table multi_tables_ddl_test.t55 not exists for 1-th check, retry later chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/synced_status Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... Verifying downstream PD is started... table multi_tables_ddl_test.t55 not exists for 2-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/kafka_column_selector_avro/run.sh using Sink-Type: mysql... <<================= [Sun Apr 6 08:30:19 CST 2025] <<<<<< run test case kafka_column_selector_avro success! >>>>>> Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table multi_tables_ddl_test.t55 not exists for 3-th check, retry later Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table multi_tables_ddl_test.t55 not exists for 4-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/debezium/run.sh using Sink-Type: mysql... <<================= =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/lossy_ddl/run.sh using Sink-Type: mysql... <<================= [Sun Apr 6 08:30:23 CST 2025] <<<<<< run test case lossy_ddl success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table multi_tables_ddl_test.t55 not exists for 5-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489990959950498966 TiDB Cluster ID. tikv_gc_leader_uuid 6582205d7cc000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-c-g3td9, pid:18823, start at 2025-04-06 08:30:24.273899675 +0800 CST m=+2.381170443 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:32:24.283 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:30:24.243 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:20:24.243 +0800 All versions after safe point can be accessed. (DO NOT EDIT) table multi_tables_ddl_test.t55 exists table multi_tables_ddl_test.t66 exists table multi_tables_ddl_test.t7 exists table multi_tables_ddl_test.t88 exists table multi_tables_ddl_test.finish_mark not exists for 1-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489990959950498966 TiDB Cluster ID. tikv_gc_leader_uuid 6582205d7cc000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-c-g3td9, pid:18823, start at 2025-04-06 08:30:24.273899675 +0800 CST m=+2.381170443 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:32:24.283 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:30:24.243 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:20:24.243 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489990971315334978 TiDB Cluster ID. tikv_gc_leader_uuid 6582205d7d40002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-c-g3td9, pid:18897, start at 2025-04-06 08:30:24.246842312 +0800 CST m=+2.291385273 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:32:24.254 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:30:24.245 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:20:24.245 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/synced_status/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/synced_status/tiflash/log/error.log arg matches is ArgMatches { args: {"log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash-proxy.toml"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash/db/proxy"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/storage_csv_update/run.sh using Sink-Type: mysql... <<================= [Sun Apr 6 08:30:27 CST 2025] <<<<<< run test case storage_csv_update success! >>>>>> table multi_tables_ddl_test.finish_mark not exists for 2-th check, retry later + cd /tmp/tidb_cdc_test/synced_status ++ run_cdc_cli_tso_query 127.0.0.1 2379 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.20144.out cli tso query --pd=http://127.0.0.1:2379 + set +x + tso='457152771809083393 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 457152771809083393 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + start_ts=457152771809083393 + run_cdc_server --workdir /tmp/tidb_cdc_test/synced_status --binary cdc.test [Sun Apr 6 08:30:30 CST 2025] <<<<<< START cdc server in synced_status case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.2017820180.out server --log-file /tmp/tidb_cdc_test/synced_status/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/synced_status/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table multi_tables_ddl_test.finish_mark exists check table exists success + endpoints=http://127.0.0.1:2379 + changefeed_id=test-normal + expected_state=normal + error_msg=null + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c test-normal -s + info='{ "upstream_id": 7489990900010735562, "namespace": "default", "id": "test-normal", "state": "normal", "checkpoint_tso": 457152768469631041, "checkpoint_time": "2025-04-06 08:30:15.854", "error": null }' + echo '{ "upstream_id": 7489990900010735562, "namespace": "default", "id": "test-normal", "state": "normal", "checkpoint_tso": 457152768469631041, "checkpoint_time": "2025-04-06 08:30:15.854", "error": null }' { "upstream_id": 7489990900010735562, "namespace": "default", "id": "test-normal", "state": "normal", "checkpoint_tso": 457152768469631041, "checkpoint_time": "2025-04-06 08:30:15.854", "error": null } ++ echo '{' '"upstream_id":' 7489990900010735562, '"namespace":' '"default",' '"id":' '"test-normal",' '"state":' '"normal",' '"checkpoint_tso":' 457152768469631041, '"checkpoint_time":' '"2025-04-06' '08:30:15.854",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7489990900010735562, '"namespace":' '"default",' '"id":' '"test-normal",' '"state":' '"normal",' '"checkpoint_tso":' 457152768469631041, '"checkpoint_time":' '"2025-04-06' '08:30:15.854",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] + endpoints=http://127.0.0.1:2379 + changefeed_id=test-error-1 + expected_state=normal + error_msg=null + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c test-error-1 -s + info='{ "upstream_id": 7489990900010735562, "namespace": "default", "id": "test-error-1", "state": "normal", "checkpoint_tso": 457152771969253386, "checkpoint_time": "2025-04-06 08:30:29.204", "error": null }' + echo '{ "upstream_id": 7489990900010735562, "namespace": "default", "id": "test-error-1", "state": "normal", "checkpoint_tso": 457152771969253386, "checkpoint_time": "2025-04-06 08:30:29.204", "error": null }' { "upstream_id": 7489990900010735562, "namespace": "default", "id": "test-error-1", "state": "normal", "checkpoint_tso": 457152771969253386, "checkpoint_time": "2025-04-06 08:30:29.204", "error": null } ++ echo '{' '"upstream_id":' 7489990900010735562, '"namespace":' '"default",' '"id":' '"test-error-1",' '"state":' '"normal",' '"checkpoint_tso":' 457152771969253386, '"checkpoint_time":' '"2025-04-06' '08:30:29.204",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7489990900010735562, '"namespace":' '"default",' '"id":' '"test-error-1",' '"state":' '"normal",' '"checkpoint_tso":' 457152771969253386, '"checkpoint_time":' '"2025-04-06' '08:30:29.204",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] + endpoints=http://127.0.0.1:2379 + changefeed_id=test-error-2 + expected_state=failed + error_msg=ErrSyncRenameTableFailed + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c test-error-2 -s + info='{ "upstream_id": 7489990900010735562, "namespace": "default", "id": "test-error-2", "state": "failed", "checkpoint_tso": 457152767342411777, "checkpoint_time": "2025-04-06 08:30:11.554", "error": { "time": "2025-04-06T08:30:16.400458138+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrSyncRenameTableFailed", "message": "[CDC:ErrSyncRenameTableFailed]table'\''s old name is not in filter rule, and its new name in filter rule table id '\''138'\'', ddl query: [rename table t11 to t9], it'\''s an unexpected behavior, if you want to replicate this table, please add its old name to filter rule." } }' + echo '{ "upstream_id": 7489990900010735562, "namespace": "default", "id": "test-error-2", "state": "failed", "checkpoint_tso": 457152767342411777, "checkpoint_time": "2025-04-06 08:30:11.554", "error": { "time": "2025-04-06T08:30:16.400458138+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrSyncRenameTableFailed", "message": "[CDC:ErrSyncRenameTableFailed]table'\''s old name is not in filter rule, and its new name in filter rule table id '\''138'\'', ddl query: [rename table t11 to t9], it'\''s an unexpected behavior, if you want to replicate this table, please add its old name to filter rule." } }' { "upstream_id": 7489990900010735562, "namespace": "default", "id": "test-error-2", "state": "failed", "checkpoint_tso": 457152767342411777, "checkpoint_time": "2025-04-06 08:30:11.554", "error": { "time": "2025-04-06T08:30:16.400458138+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrSyncRenameTableFailed", "message": "[CDC:ErrSyncRenameTableFailed]table's old name is not in filter rule, and its new name in filter rule table id '138', ddl query: [rename table t11 to t9], it's an unexpected behavior, if you want to replicate this table, please add its old name to filter rule." } } ++ echo '{' '"upstream_id":' 7489990900010735562, '"namespace":' '"default",' '"id":' '"test-error-2",' '"state":' '"failed",' '"checkpoint_tso":' 457152767342411777, '"checkpoint_time":' '"2025-04-06' '08:30:11.554",' '"error":' '{' '"time":' '"2025-04-06T08:30:16.400458138+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrSyncRenameTableFailed",' '"message":' '"[CDC:ErrSyncRenameTableFailed]table'\''s' old name is not in filter rule, and its new name in filter rule table id ''\''138'\'',' ddl query: '[rename' table t11 to 't9],' 'it'\''s' an unexpected behavior, if you want to replicate this table, please add its old name to filter 'rule."' '}' '}' ++ jq -r .state + state=failed + [[ ! failed == \f\a\i\l\e\d ]] ++ jq -r .error.message ++ echo '{' '"upstream_id":' 7489990900010735562, '"namespace":' '"default",' '"id":' '"test-error-2",' '"state":' '"failed",' '"checkpoint_tso":' 457152767342411777, '"checkpoint_time":' '"2025-04-06' '08:30:11.554",' '"error":' '{' '"time":' '"2025-04-06T08:30:16.400458138+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrSyncRenameTableFailed",' '"message":' '"[CDC:ErrSyncRenameTableFailed]table'\''s' old name is not in filter rule, and its new name in filter rule table id ''\''138'\'',' ddl query: '[rename' table t11 to 't9],' 'it'\''s' an unexpected behavior, if you want to replicate this table, please add its old name to filter 'rule."' '}' '}' + message='[CDC:ErrSyncRenameTableFailed]table'\''s old name is not in filter rule, and its new name in filter rule table id '\''138'\'', ddl query: [rename table t11 to t9], it'\''s an unexpected behavior, if you want to replicate this table, please add its old name to filter rule.' + [[ ! [CDC:ErrSyncRenameTableFailed]table's old name is not in filter rule, and its new name in filter rule table id '138', ddl query: [rename table t11 to t9], it's an unexpected behavior, if you want to replicate this table, please add its old name to filter rule. =~ ErrSyncRenameTableFailed ]] check diff successfully ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 221 100 221 0 0 6721 0 --:--:-- --:--:-- --:--:-- 6906 + synced_status='{"synced":true,"sink_checkpoint_ts":"2025-04-06 08:30:18.181","puller_resolved_ts":"2025-04-06 08:30:13.181","last_synced_ts":"2025-04-06 08:28:04.430","now_ts":"2025-04-06 08:30:19.000","info":"Data syncing is finished"}' ++ echo '{"synced":true,"sink_checkpoint_ts":"2025-04-06' '08:30:18.181","puller_resolved_ts":"2025-04-06' '08:30:13.181","last_synced_ts":"2025-04-06' '08:28:04.430","now_ts":"2025-04-06' '08:30:19.000","info":"Data' syncing is 'finished"}' ++ jq .synced + status=true + '[' true '!=' true ']' ++ echo '{"synced":true,"sink_checkpoint_ts":"2025-04-06' '08:30:18.181","puller_resolved_ts":"2025-04-06' '08:30:13.181","last_synced_ts":"2025-04-06' '08:28:04.430","now_ts":"2025-04-06' '08:30:19.000","info":"Data' syncing is 'finished"}' ++ jq -r .info + info='Data syncing is finished' + target_message='Data syncing is finished' + '[' 'Data syncing is finished' '!=' 'Data syncing is finished' ']' + cleanup_process cdc.test wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit + stop_tidb_cluster + run_case_with_failpoint conf/changefeed-redo.toml + rm -rf /tmp/tidb_cdc_test/synced_status_with_redo + mkdir -p /tmp/tidb_cdc_test/synced_status_with_redo + start_tidb_cluster --workdir /tmp/tidb_cdc_test/synced_status_with_redo shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory The 1 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory wait process cdc.test exit for 1-th time... chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/synced_status_with_redo Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/csv_storage_update_pk_clustered/run.sh using Sink-Type: mysql... <<================= [Sun Apr 6 08:30:31 CST 2025] <<<<<< run test case csv_storage_update_pk_clustered success! >>>>>> wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Sun Apr 6 08:30:33 CST 2025] <<<<<< run test case multi_tables_ddl success! >>>>>> Exiting on signal: INTERRUPT + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 06 Apr 2025 00:30:33 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5a73bb97-3c9c-43ee-bbff-b63e6e74fae7 {"id":"5a73bb97-3c9c-43ee-bbff-b63e6e74fae7","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899430} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196088154b8eb 5a73bb97-3c9c-43ee-bbff-b63e6e74fae7 /tidb/cdc/default/default/upstream/7489990959950498966 {"id":7489990959950498966,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5a73bb97-3c9c-43ee-bbff-b63e6e74fae7 {"id":"5a73bb97-3c9c-43ee-bbff-b63e6e74fae7","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899430} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196088154b8eb 5a73bb97-3c9c-43ee-bbff-b63e6e74fae7 /tidb/cdc/default/default/upstream/7489990959950498966 {"id":7489990959950498966,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5a73bb97-3c9c-43ee-bbff-b63e6e74fae7 {"id":"5a73bb97-3c9c-43ee-bbff-b63e6e74fae7","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899430} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/223196088154b8eb 5a73bb97-3c9c-43ee-bbff-b63e6e74fae7 /tidb/cdc/default/default/upstream/7489990959950498966 {"id":7489990959950498966,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + config_path=conf/changefeed.toml + SINK_URI='mysql://root@127.0.0.1:3306/?max-txn-row=1' + run_cdc_cli changefeed create --start-ts=457152771809083393 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/synced_status/conf/changefeed.toml + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.20244.out cli changefeed create --start-ts=457152771809083393 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/synced_status/conf/changefeed.toml Create changefeed successfully! ID: test-1 Info: {"upstream_id":7489990959950498966,"namespace":"default","id":"test-1","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2025-04-06T08:30:33.515831982+08:00","start_ts":457152771809083393,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":120,"checkpoint_interval":20}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457152771809083393,"checkpoint_ts":457152771809083393,"checkpoint_time":"2025-04-06 08:30:28.593"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x + run_sql 'USE TEST;Create table t1(a int primary key, b int);insert into t1 values(1,2);insert into t1 values(2,3);' + check_table_exists test.t1 127.0.0.1 3306 table test.t1 not exists for 1-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/csv_storage_update_pk_nonclustered/run.sh using Sink-Type: mysql... <<================= [Sun Apr 6 08:30:36 CST 2025] <<<<<< run test case csv_storage_update_pk_nonclustered success! >>>>>> Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.t1 exists + sleep 5 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } [Pipeline] // timeout [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-periodics_cdc_integration_mysql_test-234/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489991035122800657 TiDB Cluster ID. tikv_gc_leader_uuid 6582206bfb4000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-d-7dnbd, pid:17894, start at 2025-04-06 08:30:39.101870874 +0800 CST m=+2.219753294 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:32:39.109 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:30:39.085 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:20:39.085 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489991035122800657 TiDB Cluster ID. tikv_gc_leader_uuid 6582206bfb4000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-d-7dnbd, pid:17894, start at 2025-04-06 08:30:39.101870874 +0800 CST m=+2.219753294 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:32:39.109 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:30:39.085 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:20:39.085 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489991029156865185 TiDB Cluster ID. tikv_gc_leader_uuid 6582206bfa8000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-d-7dnbd, pid:17962, start at 2025-04-06 08:30:39.101637217 +0800 CST m=+2.140749583 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:32:39.108 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:30:39.082 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:20:39.082 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/synced_status_with_redo/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/synced_status_with_redo/tiflash/log/error.log arg matches is ArgMatches { args: {"config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/synced_status_with_redo/tiflash-proxy.toml"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/synced_status_with_redo/tiflash/log/proxy.log"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/synced_status_with_redo/tiflash/db/proxy"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + kill_tikv ++ ps aux ++ grep tikv-server ++ grep /tmp/tidb_cdc_test/synced_status + info='jenkins 18163 25.1 0.6 4864360 2488748 ? Sl 08:30 0:05 tikv-server --pd 127.0.0.1:2379 -A 127.0.0.1:20160 --status-addr 127.0.0.1:20181 --log-file /tmp/tidb_cdc_test/synced_status/tikv1.log --log-level debug -C /tmp/tidb_cdc_test/synced_status/tikv-config.toml -s /tmp/tidb_cdc_test/synced_status/tikv1 jenkins 18164 24.9 0.6 4860780 2496752 ? Sl 08:30 0:05 tikv-server --pd 127.0.0.1:2379 -A 127.0.0.1:20161 --status-addr 127.0.0.1:20182 --log-file /tmp/tidb_cdc_test/synced_status/tikv2.log --log-level debug -C /tmp/tidb_cdc_test/synced_status/tikv-config.toml -s /tmp/tidb_cdc_test/synced_status/tikv2 jenkins 18165 34.2 0.6 4887912 2535868 ? Sl 08:30 0:07 tikv-server --pd 127.0.0.1:2379 -A 127.0.0.1:20162 --status-addr 127.0.0.1:20183 --log-file /tmp/tidb_cdc_test/synced_status/tikv3.log --log-level debug -C /tmp/tidb_cdc_test/synced_status/tikv-config.toml -s /tmp/tidb_cdc_test/synced_status/tikv3 jenkins 18167 32.8 0.6 4881260 2518396 ? Sl 08:30 0:07 tikv-server --pd 127.0.0.1:2479 -A 127.0.0.1:21160 --status-addr 127.0.0.1:21180 --log-file /tmp/tidb_cdc_test/synced_status/tikv_down.log --log-level debug -C /tmp/tidb_cdc_test/synced_status/tikv-config.toml -s /tmp/tidb_cdc_test/synced_status/tikv_down' ++ ps aux ++ grep tikv-server ++ grep /tmp/tidb_cdc_test/synced_status ++ awk '{print $2}' ++ xargs kill -9 ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 243 100 243 0 0 8329 0 --:--:-- --:--:-- --:--:-- 8379 + synced_status='{"synced":false,"sink_checkpoint_ts":"2025-04-06 08:30:40.943","puller_resolved_ts":"2025-04-06 08:30:34.993","last_synced_ts":"2025-04-06 08:30:34.993","now_ts":"2025-04-06 08:30:42.000","info":"The data syncing is not finished, please wait"}' ++ echo '{"synced":false,"sink_checkpoint_ts":"2025-04-06' '08:30:40.943","puller_resolved_ts":"2025-04-06' '08:30:34.993","last_synced_ts":"2025-04-06' '08:30:34.993","now_ts":"2025-04-06' '08:30:42.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq .synced + status=false + '[' false '!=' false ']' ++ echo '{"synced":false,"sink_checkpoint_ts":"2025-04-06' '08:30:40.943","puller_resolved_ts":"2025-04-06' '08:30:34.993","last_synced_ts":"2025-04-06' '08:30:34.993","now_ts":"2025-04-06' '08:30:42.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq -r .info + info='The data syncing is not finished, please wait' + target_message='The data syncing is not finished, please wait' + '[' 'The data syncing is not finished, please wait' '!=' 'The data syncing is not finished, please wait' ']' + sleep 130 + cd /tmp/tidb_cdc_test/synced_status_with_redo + export 'GO_FAILPOINTS=github.com/pingcap/tiflow/cdc/owner/ChangefeedOwnerNotUpdateCheckpoint=return(true)' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ChangefeedOwnerNotUpdateCheckpoint=return(true)' ++ run_cdc_cli_tso_query 127.0.0.1 2379 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status_with_redo.cli.19334.out cli tso query --pd=http://127.0.0.1:2379 + set +x + tso='457152775726039041 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 457152775726039041 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + start_ts=457152775726039041 + run_cdc_server --workdir /tmp/tidb_cdc_test/synced_status_with_redo --binary cdc.test [Sun Apr 6 08:30:44 CST 2025] <<<<<< START cdc server in synced_status_with_redo case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ChangefeedOwnerNotUpdateCheckpoint=return(true)' + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status_with_redo.1936619368.out server --log-file /tmp/tidb_cdc_test/synced_status_with_redo/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/synced_status_with_redo/cdc_data --cluster-id default + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } [Pipeline] // timeout [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-periodics_cdc_integration_mysql_test-234/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 06 Apr 2025 00:30:48 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c382e13b-8713-4864-b461-13fb537d9c7f {"id":"c382e13b-8713-4864-b461-13fb537d9c7f","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899445} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319608819364e3 c382e13b-8713-4864-b461-13fb537d9c7f /tidb/cdc/default/default/upstream/7489991035122800657 {"id":7489991035122800657,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c382e13b-8713-4864-b461-13fb537d9c7f {"id":"c382e13b-8713-4864-b461-13fb537d9c7f","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899445} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319608819364e3 c382e13b-8713-4864-b461-13fb537d9c7f /tidb/cdc/default/default/upstream/7489991035122800657 {"id":7489991035122800657,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c382e13b-8713-4864-b461-13fb537d9c7f {"id":"c382e13b-8713-4864-b461-13fb537d9c7f","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899445} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22319608819364e3 c382e13b-8713-4864-b461-13fb537d9c7f /tidb/cdc/default/default/upstream/7489991035122800657 {"id":7489991035122800657,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + config_path=conf/changefeed-redo.toml + SINK_URI='mysql://root@127.0.0.1:3306/?max-txn-row=1' + run_cdc_cli changefeed create --start-ts=457152775726039041 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/synced_status_with_redo/conf/changefeed-redo.toml + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status_with_redo.cli.19423.out cli changefeed create --start-ts=457152775726039041 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/synced_status_with_redo/conf/changefeed-redo.toml Create changefeed successfully! ID: test-1 Info: {"upstream_id":7489991035122800657,"namespace":"default","id":"test-1","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2025-04-06T08:30:48.428692042+08:00","start_ts":457152775726039041,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"eventual","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"storage":"file:///tmp/tidb_cdc_test/synced_status/redo","use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":120,"checkpoint_interval":20}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457152775726039041,"checkpoint_ts":457152775726039041,"checkpoint_time":"2025-04-06 08:30:43.535"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... + set +x + sleep 20 ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 723 100 723 0 0 14787 0 --:--:-- --:--:-- --:--:-- 15062 + synced_status='{"synced":false,"sink_checkpoint_ts":"2025-04-06 08:30:43.535","puller_resolved_ts":"1970-01-01 08:00:00.000","last_synced_ts":"1970-01-01 08:00:00.000","now_ts":"2025-04-06 08:31:09.000","info":"Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' \u003e '\''Resolved-Ts'\'' \u003e '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait"}' ++ echo '{"synced":false,"sink_checkpoint_ts":"2025-04-06' '08:30:43.535","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2025-04-06' '08:31:09.000","info":"Please' check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view ''\''TiKV-Details'\''' '\u003e' ''\''Resolved-Ts'\''' '\u003e' ''\''Max' Leader Resolved TS 'gap'\''' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please 'wait"}' ++ jq .synced + status=false + '[' false '!=' false ']' ++ echo '{"synced":false,"sink_checkpoint_ts":"2025-04-06' '08:30:43.535","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2025-04-06' '08:31:09.000","info":"Please' check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view ''\''TiKV-Details'\''' '\u003e' ''\''Resolved-Ts'\''' '\u003e' ''\''Max' Leader Resolved TS 'gap'\''' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please 'wait"}' ++ jq -r .info + info='Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' + target_message='Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' + '[' 'Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' '!=' 'Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' ']' + export GO_FAILPOINTS= + GO_FAILPOINTS= + cleanup_process cdc.test wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit + stop_tidb_cluster + check_logs /tmp/tidb_cdc_test/synced_status_with_redo ++ date + echo '[Sun Apr 6 08:31:20 CST 2025] <<<<<< run test case synced_status_with_redo success! >>>>>>' [Sun Apr 6 08:31:20 CST 2025] <<<<<< run test case synced_status_with_redo success! >>>>>> + stop_tidb_cluster \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } [Pipeline] // timeout [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-periodics_cdc_integration_mysql_test-234/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit # diff Configuration. check-thread-count = 4 export-fix-sql = true check-struct-only = false [task] output-dir = "/tmp/tidb_cdc_test/partition_table/sync_diff/output" source-instances = ["mysql1"] target-instance = "tidb0" target-check-tables = ["partition_table.?*"] [data-sources] [data-sources.mysql1] host = "127.0.0.1" port = 4000 user = "root" password = "" [data-sources.tidb0] host = "127.0.0.1" port = 3306 user = "root" password = "" [2025/04/06 08:32:16.281 +08:00] [INFO] [helper.go:54] ["init log"] [file=] [level=info] [2025/04/06 08:32:16.282 +08:00] [INFO] [apply.go:78] ["set memory limit"] [memoryLimit=10737418240] [2025/04/06 08:32:16.282 +08:00] [INFO] [apply.go:94] ["Start http pprof server"] [addr=127.0.0.1:6060] [2025/04/06 08:32:16.283 +08:00] [INFO] [mem_quota.go:82] ["New memory quota"] [namespace=default] [changefeed=redo-applier] [total=1073741824] [2025/04/06 08:32:16.283 +08:00] [INFO] [redo.go:167] ["apply redo log starts"] [checkpointTs=457152765925523461] [resolvedTs=457152798994989059] [2025/04/06 08:32:16.284 +08:00] [INFO] [tz.go:35] ["Use the timezone of the TiCDC server machine"] [timezoneName=System] [timezone=Asia/Shanghai] [2025/04/06 08:32:16.284 +08:00] [WARN] [config.go:428] ["Because time-zone is not specified, the timezone of the TiCDC server will be used. We recommend that you specify the time-zone explicitly. Please make sure that the timezone of the TiCDC server, sink-uri and the downstream database are consistent. If the downstream database does not load the timezone information, you can refer to https://dev.mysql.com/doc/refman/8.0/en/mysql-tzinfo-to-sql.html."] [timezone=Asia/Shanghai] [2025/04/06 08:32:16.284 +08:00] [ERROR] [config.go:216] ["The TiDB source ID should never be set to 0. Please report it as a bug. The default value will be used: 1."] [tidbSourceID=0] [2025/04/06 08:32:16.284 +08:00] [INFO] [file.go:114] ["succeed to download and sort redo logs"] [type=ddl] [duration=1.412272ms] [2025/04/06 08:32:16.285 +08:00] [INFO] [file.go:114] ["succeed to download and sort redo logs"] [type=row] [duration=1.821545ms] [2025/04/06 08:32:16.288 +08:00] [INFO] [db_helper.go:161] ["sink uri is configured"] [dsn="normal:******@tcp(127.0.0.1:3306)/?interpolateParams=true&multiStatements=true&allow_auto_random_explicit_insert=1&charset=utf8mb4&foreign_key_checks=0&maxAllowedPacket=0&readTimeout=2m&sql_mode=%22IGNORE_SPACE%2CALLOW_INVALID_DATES%2CNO_AUTO_VALUE_ON_ZERO%2CNO_ENGINE_SUBSTITUTION%2CONLY_FULL_GROUP_BY%22&tidb_enable_external_ts_read=%22OFF%22&tidb_placement_mode=%22ignore%22&tidb_txn_mode=optimistic&time_zone=%22Asia%2FShanghai%22&timeout=2m&transaction_isolation=%22READ-COMMITTED%22&writeTimeout=2m"] [2025/04/06 08:32:16.291 +08:00] [INFO] [mysql.go:195] ["MySQL backends is created"] [changefeed=.] [workerCount=16] [forceReplicate=false] [2025/04/06 08:32:16.293 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/06 08:32:16.293 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/06 08:32:16.293 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/06 08:32:16.293 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/06 08:32:16.293 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/06 08:32:16.293 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/06 08:32:16.293 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/06 08:32:16.293 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/06 08:32:16.293 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/06 08:32:16.293 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/06 08:32:16.293 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/06 08:32:16.293 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/06 08:32:16.293 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/06 08:32:16.293 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/06 08:32:16.293 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/06 08:32:16.293 +08:00] [INFO] [txn_cache.go:71] ["create new worker cache in conflict detector"] [cacheCount=16] [cacheSize=1024] [BlockStrategy=waitEmpty] [2025/04/06 08:32:16.293 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=1] [2025/04/06 08:32:16.293 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=0] [2025/04/06 08:32:16.293 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=3] [2025/04/06 08:32:16.293 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=4] [2025/04/06 08:32:16.293 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=2] [2025/04/06 08:32:16.293 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=7] [2025/04/06 08:32:16.293 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=5] [2025/04/06 08:32:16.293 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=6] [2025/04/06 08:32:16.293 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=8] [2025/04/06 08:32:16.293 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=9] [2025/04/06 08:32:16.293 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=11] [2025/04/06 08:32:16.293 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=10] [2025/04/06 08:32:16.293 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=12] [2025/04/06 08:32:16.293 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=14] [2025/04/06 08:32:16.293 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=15] [2025/04/06 08:32:16.293 +08:00] [INFO] [worker.go:90] ["Transaction dmlSink worker starts"] [changefeedID=.] [workerID=13] [2025/04/06 08:32:16.293 +08:00] [INFO] [tz.go:35] ["Use the timezone of the TiCDC server machine"] [timezoneName=System] [timezone=Asia/Shanghai] [2025/04/06 08:32:16.293 +08:00] [WARN] [config.go:428] ["Because time-zone is not specified, the timezone of the TiCDC server will be used. We recommend that you specify the time-zone explicitly. Please make sure that the timezone of the TiCDC server, sink-uri and the downstream database are consistent. If the downstream database does not load the timezone information, you can refer to https://dev.mysql.com/doc/refman/8.0/en/mysql-tzinfo-to-sql.html."] [timezone=Asia/Shanghai] [2025/04/06 08:32:16.293 +08:00] [ERROR] [config.go:216] ["The TiDB source ID should never be set to 0. Please report it as a bug. The default value will be used: 1."] [tidbSourceID=0] [2025/04/06 08:32:16.297 +08:00] [INFO] [db_helper.go:161] ["sink uri is configured"] [dsn="normal:******@tcp(127.0.0.1:3306)/?interpolateParams=true&multiStatements=true&allow_auto_random_explicit_insert=1&charset=utf8mb4&foreign_key_checks=0&maxAllowedPacket=0&readTimeout=2m&sql_mode=%22IGNORE_SPACE%2CALLOW_INVALID_DATES%2CNO_AUTO_VALUE_ON_ZERO%2CNO_ENGINE_SUBSTITUTION%2CONLY_FULL_GROUP_BY%22&tidb_enable_external_ts_read=%22OFF%22&tidb_placement_mode=%22ignore%22&tidb_txn_mode=optimistic&time_zone=%22Asia%2FShanghai%22&timeout=2m&transaction_isolation=%22READ-COMMITTED%22&writeTimeout=2m"] [2025/04/06 08:32:16.299 +08:00] [WARN] [mysql_ddl_sink.go:291] ["please set `has-vector-type` to be true if a column is vector type when the downstream is not TiDB or TiDB version less than specify version"] [hasVectorType=false] [supportVectorVersion=8.4.0] [2025/04/06 08:32:16.299 +08:00] [INFO] [mysql_ddl_sink.go:122] ["MySQL DDL sink is created"] [namespace=] [changefeed=] [2025/04/06 08:32:16.299 +08:00] [WARN] [redo.go:287] ["apply DDL"] [ddl="{\"StartTs\":457152767471910934,\"CommitTs\":457152767471910938,\"Query\":\"alter table t1 drop partition p1\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"Schema\":\"partition_table\",\"Table\":\"t1\",\"TableID\":121,\"IsPartition\":true},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":20,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false,\"BDRRole\":\"\",\"SQLMode\":0}"] [2025/04/06 08:32:18.307 +08:00] [ERROR] [async_ddl.go:184] ["check async exec ddl failed"] [namespace=] [changefeed=] [error="Error 8121 (HY000): privilege check for 'Super' fail"] [2025/04/06 08:32:18.307 +08:00] [INFO] [mysql_ddl_sink.go:225] ["Start exec DDL"] [namespace=] [changefeed=] [commitTs=457152767471910938] [DDL="alter table t1 drop partition p1"] [2025/04/06 08:32:18.372 +08:00] [INFO] [mysql_ddl_sink.go:272] ["Exec DDL succeeded"] [namespace=] [changefeed=] [duration=64.759497ms] [sql="alter table t1 drop partition p1"] [2025/04/06 08:32:18.372 +08:00] [WARN] [redo.go:287] ["apply DDL"] [ddl="{\"StartTs\":457152767602982915,\"CommitTs\":457152767602982920,\"Query\":\"ALTER TABLE `partition_table`.`t1` EXCHANGE PARTITION `p3` WITH TABLE `partition_table`.`t2`\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"Schema\":\"partition_table\",\"Table\":\"t1\",\"TableID\":121,\"IsPartition\":true},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":42,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false,\"BDRRole\":\"\",\"SQLMode\":0}"] [2025/04/06 08:32:18.573 +08:00] [INFO] [mysql_ddl_sink.go:225] ["Start exec DDL"] [namespace=] [changefeed=] [commitTs=457152767602982920] [DDL="ALTER TABLE `partition_table`.`t1` EXCHANGE PARTITION `p3` WITH TABLE `partition_table`.`t2`"] [2025/04/06 08:32:18.615 +08:00] [INFO] [mysql_ddl_sink.go:272] ["Exec DDL succeeded"] [namespace=] [changefeed=] [duration=41.920729ms] [sql="ALTER TABLE `partition_table`.`t1` EXCHANGE PARTITION `p3` WITH TABLE `partition_table`.`t2`"] [2025/04/06 08:32:18.615 +08:00] [WARN] [redo.go:287] ["apply DDL"] [ddl="{\"StartTs\":457152767734317080,\"CommitTs\":457152767734317084,\"Query\":\"ALTER TABLE `partition_table`.`t1` EXCHANGE PARTITION `p3` WITH TABLE `partition_table2`.`t2`\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"Schema\":\"partition_table\",\"Table\":\"t1\",\"TableID\":121,\"IsPartition\":true},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":42,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false,\"BDRRole\":\"\",\"SQLMode\":0}"] [2025/04/06 08:32:19.015 +08:00] [INFO] [mysql_ddl_sink.go:225] ["Start exec DDL"] [namespace=] [changefeed=] [commitTs=457152767734317084] [DDL="ALTER TABLE `partition_table`.`t1` EXCHANGE PARTITION `p3` WITH TABLE `partition_table2`.`t2`"] [2025/04/06 08:32:19.055 +08:00] [INFO] [mysql_ddl_sink.go:272] ["Exec DDL succeeded"] [namespace=] [changefeed=] [duration=39.382419ms] [sql="ALTER TABLE `partition_table`.`t1` EXCHANGE PARTITION `p3` WITH TABLE `partition_table2`.`t2`"] [2025/04/06 08:32:19.055 +08:00] [WARN] [redo.go:287] ["apply DDL"] [ddl="{\"StartTs\":457152768009306138,\"CommitTs\":457152768009306141,\"Query\":\"ALTER TABLE t1 REORGANIZE PARTITION p0,p2 INTO (PARTITION p0 VALUES LESS THAN (5), PARTITION p1 VALUES LESS THAN (10), PARTITION p2 VALUES LESS THAN (21))\",\"TableInfo\":{\"SchemaID\":0,\"TableName\":{\"Schema\":\"partition_table\",\"Table\":\"t1\",\"TableID\":121,\"IsPartition\":true},\"Version\":0,\"RowColumnsOffset\":null,\"ColumnsFlag\":null,\"HandleIndexID\":0,\"IndexColumnsOffset\":null},\"PreTableInfo\":null,\"Type\":64,\"Done\":{},\"Charset\":\"\",\"Collate\":\"\",\"IsBootstrap\":false,\"BDRRole\":\"\",\"SQLMode\":0}"] [2025/04/06 08:32:19.457 +08:00] [INFO] [mysql_ddl_sink.go:225] ["Start exec DDL"] [namespace=] [changefeed=] [commitTs=457152768009306141] [DDL="ALTER TABLE t1 REORGANIZE PARTITION p0,p2 INTO (PARTITION p0 VALUES LESS THAN (5), PARTITION p1 VALUES LESS THAN (10), PARTITION p2 VALUES LESS THAN (21))"] [2025/04/06 08:32:20.298 +08:00] [INFO] [mysql_ddl_sink.go:272] ["Exec DDL succeeded"] [namespace=] [changefeed=] [duration=841.058223ms] [sql="ALTER TABLE t1 REORGANIZE PARTITION p0,p2 INTO (PARTITION p0 VALUES LESS THAN (5), PARTITION p1 VALUES LESS THAN (10), PARTITION p2 VALUES LESS THAN (21))"] [2025/04/06 08:32:20.298 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [span="{tableID:119, startKey:7480000000000000ff775f720000000000fa, endKey:7480000000000000ff775f730000000000fa}"] [checkpointTs=457152798994989059] [2025/04/06 08:32:20.298 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [span="{tableID:119, startKey:7480000000000000ff775f720000000000fa, endKey:7480000000000000ff775f730000000000fa}"] [checkpointTs=457152798994989059] [2025/04/06 08:32:20.398 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [span="{tableID:117, startKey:7480000000000000ff755f720000000000fa, endKey:7480000000000000ff755f730000000000fa}"] [checkpointTs=457152798994989059] [2025/04/06 08:32:20.398 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [span="{tableID:117, startKey:7480000000000000ff755f720000000000fa, endKey:7480000000000000ff755f730000000000fa}"] [checkpointTs=457152798994989059] [2025/04/06 08:32:20.498 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [span="{tableID:115, startKey:7480000000000000ff735f720000000000fa, endKey:7480000000000000ff735f730000000000fa}"] [checkpointTs=457152798994989059] [2025/04/06 08:32:20.499 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [span="{tableID:115, startKey:7480000000000000ff735f720000000000fa, endKey:7480000000000000ff735f730000000000fa}"] [checkpointTs=457152798994989059] [2025/04/06 08:32:20.599 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [span="{tableID:123, startKey:7480000000000000ff7b5f720000000000fa, endKey:7480000000000000ff7b5f730000000000fa}"] [checkpointTs=457152798994989059] [2025/04/06 08:32:20.599 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [span="{tableID:123, startKey:7480000000000000ff7b5f720000000000fa, endKey:7480000000000000ff7b5f730000000000fa}"] [checkpointTs=457152798994989059] [2025/04/06 08:32:20.699 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [span="{tableID:124, startKey:7480000000000000ff7c5f720000000000fa, endKey:7480000000000000ff7c5f730000000000fa}"] [checkpointTs=457152798994989059] [2025/04/06 08:32:20.699 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [span="{tableID:124, startKey:7480000000000000ff7c5f720000000000fa, endKey:7480000000000000ff7c5f730000000000fa}"] [checkpointTs=457152798994989059] [2025/04/06 08:32:20.799 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [span="{tableID:118, startKey:7480000000000000ff765f720000000000fa, endKey:7480000000000000ff765f730000000000fa}"] [checkpointTs=457152798994989059] [2025/04/06 08:32:20.799 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [span="{tableID:118, startKey:7480000000000000ff765f720000000000fa, endKey:7480000000000000ff765f730000000000fa}"] [checkpointTs=457152798994989059] [2025/04/06 08:32:20.899 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [span="{tableID:116, startKey:7480000000000000ff745f720000000000fa, endKey:7480000000000000ff745f730000000000fa}"] [checkpointTs=457152798994989059] [2025/04/06 08:32:20.899 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [span="{tableID:116, startKey:7480000000000000ff745f720000000000fa, endKey:7480000000000000ff745f730000000000fa}"] [checkpointTs=457152798994989059] [2025/04/06 08:32:20.999 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [span="{tableID:122, startKey:7480000000000000ff7a5f720000000000fa, endKey:7480000000000000ff7a5f730000000000fa}"] [checkpointTs=457152798994989059] [2025/04/06 08:32:21.000 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [span="{tableID:122, startKey:7480000000000000ff7a5f720000000000fa, endKey:7480000000000000ff7a5f730000000000fa}"] [checkpointTs=457152798994989059] [2025/04/06 08:32:21.100 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [span="{tableID:126, startKey:7480000000000000ff7e5f720000000000fa, endKey:7480000000000000ff7e5f730000000000fa}"] [checkpointTs=457152798994989059] [2025/04/06 08:32:21.100 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [span="{tableID:126, startKey:7480000000000000ff7e5f720000000000fa, endKey:7480000000000000ff7e5f730000000000fa}"] [checkpointTs=457152798994989059] [2025/04/06 08:32:21.200 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [span="{tableID:127, startKey:7480000000000000ff7f5f720000000000fa, endKey:7480000000000000ff7f5f730000000000fa}"] [checkpointTs=457152798994989059] [2025/04/06 08:32:21.200 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [span="{tableID:127, startKey:7480000000000000ff7f5f720000000000fa, endKey:7480000000000000ff7f5f730000000000fa}"] [checkpointTs=457152798994989059] [2025/04/06 08:32:21.300 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [span="{tableID:129, startKey:7480000000000000ff815f720000000000fa, endKey:7480000000000000ff815f730000000000fa}"] [checkpointTs=457152798994989059] [2025/04/06 08:32:21.300 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [span="{tableID:129, startKey:7480000000000000ff815f720000000000fa, endKey:7480000000000000ff815f730000000000fa}"] [checkpointTs=457152798994989059] [2025/04/06 08:32:21.400 +08:00] [INFO] [table_sink_impl.go:257] ["Table sink stopped"] [namespace=default] [changefeed=redo-applier] [span="{tableID:133, startKey:7480000000000000ff855f720000000000fa, endKey:7480000000000000ff855f730000000000fa}"] [checkpointTs=457152798994989059] [2025/04/06 08:32:21.400 +08:00] [INFO] [table_sink_impl.go:239] ["Stopping table sink"] [namespace=default] [changefeed=redo-applier] [span="{tableID:133, startKey:7480000000000000ff855f720000000000fa, endKey:7480000000000000ff855f730000000000fa}"] [checkpointTs=457152798994989059] [2025/04/06 08:32:21.501 +08:00] [INFO] [redo.go:223] ["apply redo log finishes"] [appliedLogCount=47] [appliedDDLCount=4] [currentCheckpoint=457152798994989059] [2025/04/06 08:32:21.501 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=11] [2025/04/06 08:32:21.501 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=13] [2025/04/06 08:32:21.501 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=8] [2025/04/06 08:32:21.501 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=2] [2025/04/06 08:32:21.501 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=10] [2025/04/06 08:32:21.501 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=4] [2025/04/06 08:32:21.501 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=1] [2025/04/06 08:32:21.501 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=15] [2025/04/06 08:32:21.501 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=12] [2025/04/06 08:32:21.501 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=3] [2025/04/06 08:32:21.501 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=5] [2025/04/06 08:32:21.501 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=0] [2025/04/06 08:32:21.501 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=9] [2025/04/06 08:32:21.501 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=7] [2025/04/06 08:32:21.501 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=14] [2025/04/06 08:32:21.501 +08:00] [INFO] [worker.go:101] ["Transaction dmlSink worker exits as canceled"] [changefeedID=.] [workerID=6] Apply redo log successfully check diff successfully [Sun Apr 6 08:32:21 CST 2025] <<<<<< run test case consistent_partition_table success! >>>>>> Database INFORMATION_SCHEMA METRICS_SCHEMA PERFORMANCE_SCHEMA mysql partition_table partition_table2 sys test =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/kafka_big_messages_v2/run.sh using Sink-Type: mysql... <<================= [Sun Apr 6 08:32:31 CST 2025] <<<<<< run test case kafka_big_messages_v2 success! >>>>>> =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/multi_tables_ddl_v2/run.sh using Sink-Type: mysql... <<================= [Sun Apr 6 08:32:34 CST 2025] <<<<<< run test case multi_tables_ddl_v2 success! >>>>>> =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/multi_topics_v2/run.sh using Sink-Type: mysql... <<================= [Sun Apr 6 08:32:38 CST 2025] <<<<<< run test case multi_topics_v2 success! >>>>>> =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/storage_cleanup/run.sh using Sink-Type: mysql... <<================= +++ dirname /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/storage_cleanup/run.sh ++ cd /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/storage_cleanup ++ pwd + CUR=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/storage_cleanup + source /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/storage_cleanup/../_utils/test_prepare ++ UP_TIDB_HOST=127.0.0.1 ++ UP_TIDB_PORT=4000 ++ UP_TIDB_OTHER_PORT=4001 ++ UP_TIDB_STATUS=10080 ++ UP_TIDB_OTHER_STATUS=10081 ++ DOWN_TIDB_HOST=127.0.0.1 ++ DOWN_TIDB_PORT=3306 ++ DOWN_TIDB_STATUS=20080 ++ TLS_TIDB_HOST=127.0.0.1 ++ TLS_TIDB_PORT=3307 ++ TLS_TIDB_STATUS=30080 ++ UP_PD_HOST_1=127.0.0.1 ++ UP_PD_PORT_1=2379 ++ UP_PD_PEER_PORT_1=2380 ++ UP_PD_HOST_2=127.0.0.1 ++ UP_PD_PORT_2=2679 ++ UP_PD_PEER_PORT_2=2680 ++ UP_PD_HOST_3=127.0.0.1 ++ UP_PD_PORT_3=2779 ++ UP_PD_PEER_PORT_3=2780 ++ DOWN_PD_HOST=127.0.0.1 ++ DOWN_PD_PORT=2479 ++ DOWN_PD_PEER_PORT=2480 ++ TLS_PD_HOST=127.0.0.1 ++ TLS_PD_PORT=2579 ++ TLS_PD_PEER_PORT=2580 ++ UP_TIKV_HOST_1=127.0.0.1 ++ UP_TIKV_PORT_1=20160 ++ UP_TIKV_STATUS_PORT_1=20181 ++ UP_TIKV_HOST_2=127.0.0.1 ++ UP_TIKV_PORT_2=20161 ++ UP_TIKV_STATUS_PORT_2=20182 ++ UP_TIKV_HOST_3=127.0.0.1 ++ UP_TIKV_PORT_3=20162 ++ UP_TIKV_STATUS_PORT_3=20183 ++ DOWN_TIKV_HOST=127.0.0.1 ++ DOWN_TIKV_PORT=21160 ++ DOWN_TIKV_STATUS_PORT=21180 ++ TLS_TIKV_HOST=127.0.0.1 ++ TLS_TIKV_PORT=22160 ++ TLS_TIKV_STATUS_PORT=22180 +++ cat /tmp/tidb_cdc_test/KAFKA_VERSION +++ echo 2.4.1 ++ KAFKA_VERSION=2.4.1 + WORK_DIR=/tmp/tidb_cdc_test/storage_cleanup + CDC_BINARY=cdc.test + SINK_TYPE=mysql + EXIST_FILES=() + CLEANED_FILES=() + trap stop_tidb_cluster EXIT + run mysql + '[' mysql '!=' storage ']' + return + check_logs /tmp/tidb_cdc_test/storage_cleanup ++ date + echo '[Sun Apr 6 08:32:41 CST 2025] <<<<<< run test case storage_cleanup success! >>>>>>' [Sun Apr 6 08:32:41 CST 2025] <<<<<< run test case storage_cleanup success! >>>>>> + stop_tidb_cluster =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/csv_storage_basic/run.sh using Sink-Type: mysql... <<================= [Sun Apr 6 08:32:45 CST 2025] <<<<<< run test case csv_storage_basic success! >>>>>> =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/csv_storage_multi_tables_ddl/run.sh using Sink-Type: mysql... <<================= [Sun Apr 6 08:32:48 CST 2025] <<<<<< run test case csv_storage_multi_tables_ddl success! >>>>>> =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/csv_storage_partition_table/run.sh using Sink-Type: mysql... <<================= [Sun Apr 6 08:32:51 CST 2025] <<<<<< run test case csv_storage_partition_table success! >>>>>> \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } [Pipeline] // timeout [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-periodics_cdc_integration_mysql_test-234/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 723 100 723 0 0 43919 0 --:--:-- --:--:-- --:--:-- 45187 + synced_status='{"synced":false,"sink_checkpoint_ts":"2025-04-06 08:30:41.943","puller_resolved_ts":"2025-04-06 08:30:41.943","last_synced_ts":"2025-04-06 08:30:34.993","now_ts":"2025-04-06 08:32:52.000","info":"Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' \u003e '\''Resolved-Ts'\'' \u003e '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait"}' ++ jq .synced ++ echo '{"synced":false,"sink_checkpoint_ts":"2025-04-06' '08:30:41.943","puller_resolved_ts":"2025-04-06' '08:30:41.943","last_synced_ts":"2025-04-06' '08:30:34.993","now_ts":"2025-04-06' '08:32:52.000","info":"Please' check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view ''\''TiKV-Details'\''' '\u003e' ''\''Resolved-Ts'\''' '\u003e' ''\''Max' Leader Resolved TS 'gap'\''' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please 'wait"}' + status=false + '[' false '!=' false ']' ++ echo '{"synced":false,"sink_checkpoint_ts":"2025-04-06' '08:30:41.943","puller_resolved_ts":"2025-04-06' '08:30:41.943","last_synced_ts":"2025-04-06' '08:30:34.993","now_ts":"2025-04-06' '08:32:52.000","info":"Please' check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view ''\''TiKV-Details'\''' '\u003e' ''\''Resolved-Ts'\''' '\u003e' ''\''Max' Leader Resolved TS 'gap'\''' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please 'wait"}' ++ jq -r .info + info='Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' + target_message='Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' + '[' 'Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' '!=' 'Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' ']' + cleanup_process cdc.test wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit + stop_tidb_cluster + run_case_with_unavailable_tidb conf/changefeed.toml + rm -rf /tmp/tidb_cdc_test/synced_status + mkdir -p /tmp/tidb_cdc_test/synced_status + start_tidb_cluster --workdir /tmp/tidb_cdc_test/synced_status shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory The 1 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/synced_status Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489991696137458975 TiDB Cluster ID. tikv_gc_leader_uuid 65822119e50000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-c-g3td9, pid:21491, start at 2025-04-06 08:33:37.195151444 +0800 CST m=+24.088876694 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:35:37.203 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:33:37.172 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:23:37.172 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489991696137458975 TiDB Cluster ID. tikv_gc_leader_uuid 65822119e50000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-c-g3td9, pid:21491, start at 2025-04-06 08:33:37.195151444 +0800 CST m=+24.088876694 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:35:37.203 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:33:37.172 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:23:37.172 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489991707158094617 TiDB Cluster ID. tikv_gc_leader_uuid 65822103eb0000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-c-g3td9, pid:21564, start at 2025-04-06 08:33:14.687181439 +0800 CST m=+1.523612684 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:35:14.694 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:33:14.668 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:23:14.668 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/synced_status/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/synced_status/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash-proxy.toml"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash/log/proxy.log"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash/db/proxy"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + cd /tmp/tidb_cdc_test/synced_status ++ run_cdc_cli_tso_query 127.0.0.1 2379 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.22895.out cli tso query --pd=http://127.0.0.1:2379 + set +x + tso='457152822476013569 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 457152822476013569 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + start_ts=457152822476013569 + run_cdc_server --workdir /tmp/tidb_cdc_test/synced_status --binary cdc.test [Sun Apr 6 08:33:43 CST 2025] <<<<<< START cdc server in synced_status case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.2292222924.out server --log-file /tmp/tidb_cdc_test/synced_status/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/synced_status/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 06 Apr 2025 00:33:46 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/faf025d4-14d2-40c1-a31d-dcf060e7fdf6 {"id":"faf025d4-14d2-40c1-a31d-dcf060e7fdf6","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899623} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/2231960883f1984e faf025d4-14d2-40c1-a31d-dcf060e7fdf6 /tidb/cdc/default/default/upstream/7489991696137458975 {"id":7489991696137458975,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/faf025d4-14d2-40c1-a31d-dcf060e7fdf6 {"id":"faf025d4-14d2-40c1-a31d-dcf060e7fdf6","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899623} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/2231960883f1984e faf025d4-14d2-40c1-a31d-dcf060e7fdf6 /tidb/cdc/default/default/upstream/7489991696137458975 {"id":7489991696137458975,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/faf025d4-14d2-40c1-a31d-dcf060e7fdf6 {"id":"faf025d4-14d2-40c1-a31d-dcf060e7fdf6","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899623} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/2231960883f1984e faf025d4-14d2-40c1-a31d-dcf060e7fdf6 /tidb/cdc/default/default/upstream/7489991696137458975 {"id":7489991696137458975,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + config_path=conf/changefeed.toml + SINK_URI='mysql://root@127.0.0.1:3306/?max-txn-row=1' + run_cdc_cli changefeed create --start-ts=457152822476013569 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/synced_status/conf/changefeed.toml + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.22977.out cli changefeed create --start-ts=457152822476013569 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/synced_status/conf/changefeed.toml Create changefeed successfully! ID: test-1 Info: {"upstream_id":7489991696137458975,"namespace":"default","id":"test-1","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2025-04-06T08:33:46.824553565+08:00","start_ts":457152822476013569,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":120,"checkpoint_interval":20}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457152822476013569,"checkpoint_ts":457152822476013569,"checkpoint_time":"2025-04-06 08:33:41.872"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x + run_sql 'USE TEST;Create table t1(a int primary key, b int);insert into t1 values(1,2);insert into t1 values(2,3);' + check_table_exists test.t1 127.0.0.1 3306 table test.t1 not exists for 1-th check, retry later table test.t1 exists + sleep 5 + kill_tidb ++ ps aux ++ grep tidb-server ++ grep /tmp/tidb_cdc_test/synced_status + info='jenkins 21491 5.1 0.0 2652088 192960 ? Sl 08:33 0:02 tidb-server -P 4000 -config /tmp/tidb_cdc_test/synced_status/tidb-config-1743899593099108908.toml --store tikv --path 127.0.0.1:2379 --status=10080 --log-file /tmp/tidb_cdc_test/synced_status/tidb.log jenkins 21495 2.3 0.0 2207520 158400 ? Sl 08:33 0:00 tidb-server -P 4001 -config /tmp/tidb_cdc_test/synced_status/tidb-config-1743899593102239706.toml --store tikv --path 127.0.0.1:2379 --status=10081 --log-file /tmp/tidb_cdc_test/synced_status/tidb_other.log jenkins 21564 5.7 0.0 2751708 234432 ? Sl 08:33 0:02 tidb-server -P 3306 -config /tmp/tidb_cdc_test/synced_status/tidb-config-1743899593156570357.toml --store tikv --path 127.0.0.1:2479 --status=20080 --log-file /tmp/tidb_cdc_test/synced_status/tidb_down.log' ++ ps aux ++ grep tidb-server ++ grep /tmp/tidb_cdc_test/synced_status ++ awk '{print $2}' ++ xargs kill -9 ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 243 100 243 0 0 6323 0 --:--:-- --:--:-- --:--:-- 6394 + synced_status='{"synced":false,"sink_checkpoint_ts":"2025-04-06 08:33:54.172","puller_resolved_ts":"2025-04-06 08:33:48.273","last_synced_ts":"2025-04-06 08:33:48.323","now_ts":"2025-04-06 08:33:55.000","info":"The data syncing is not finished, please wait"}' ++ echo '{"synced":false,"sink_checkpoint_ts":"2025-04-06' '08:33:54.172","puller_resolved_ts":"2025-04-06' '08:33:48.273","last_synced_ts":"2025-04-06' '08:33:48.323","now_ts":"2025-04-06' '08:33:55.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq .synced + status=false + '[' false '!=' false ']' ++ echo '{"synced":false,"sink_checkpoint_ts":"2025-04-06' '08:33:54.172","puller_resolved_ts":"2025-04-06' '08:33:48.273","last_synced_ts":"2025-04-06' '08:33:48.323","now_ts":"2025-04-06' '08:33:55.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq -r .info + info='The data syncing is not finished, please wait' + target_message='The data syncing is not finished, please wait' + '[' 'The data syncing is not finished, please wait' '!=' 'The data syncing is not finished, please wait' ']' + sleep 130 ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 221 100 221 0 0 7057 0 --:--:-- --:--:-- --:--:-- 7129 + synced_status='{"synced":true,"sink_checkpoint_ts":"2025-04-06 08:36:04.372","puller_resolved_ts":"2025-04-06 08:35:57.373","last_synced_ts":"2025-04-06 08:33:48.323","now_ts":"2025-04-06 08:36:05.000","info":"Data syncing is finished"}' ++ echo '{"synced":true,"sink_checkpoint_ts":"2025-04-06' '08:36:04.372","puller_resolved_ts":"2025-04-06' '08:35:57.373","last_synced_ts":"2025-04-06' '08:33:48.323","now_ts":"2025-04-06' '08:36:05.000","info":"Data' syncing is 'finished"}' ++ jq .synced + status=true + '[' true '!=' true ']' ++ echo '{"synced":true,"sink_checkpoint_ts":"2025-04-06' '08:36:04.372","puller_resolved_ts":"2025-04-06' '08:35:57.373","last_synced_ts":"2025-04-06' '08:33:48.323","now_ts":"2025-04-06' '08:36:05.000","info":"Data' syncing is 'finished"}' ++ jq -r .info + info='Data syncing is finished' + target_message='Data syncing is finished' + '[' 'Data syncing is finished' '!=' 'Data syncing is finished' ']' + cleanup_process cdc.test wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit + stop_tidb_cluster + run_case_with_failpoint conf/changefeed.toml + rm -rf /tmp/tidb_cdc_test/synced_status + mkdir -p /tmp/tidb_cdc_test/synced_status + start_tidb_cluster --workdir /tmp/tidb_cdc_test/synced_status shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory The 1 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/synced_status Starting Upstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Starting Downstream PD... Release Version: v9.0.0-beta.1.pre-91-g9d6bbb852 Edition: Community Git Commit Hash: 9d6bbb8529d9d2acb9cf324a65e94ece465f6346 Git Branch: master UTC Build Time: 2025-04-03 07:28:59 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 9.0.0-beta.1 Edition: Community Git Commit Hash: 3eb3439fd88f0db9de06ef92e845e80b32c9b49a Git Commit Branch: master UTC Build Time: 2025-04-02 02:57:52 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v9.0.0-beta.1.pre-511-gd9fb3a5337 Edition: Community Git Commit Hash: d9fb3a53376157be49b5a7941e6e689a33c81eb0 Git Branch: master UTC Build Time: 2025-04-03 11:14:01 GoVersion: go1.23.8 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489992536418697045 TiDB Cluster ID. tikv_gc_leader_uuid 658221c1588000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-c-g3td9, pid:24209, start at 2025-04-06 08:36:28.672989583 +0800 CST m=+2.178983600 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:38:28.680 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:36:28.642 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:26:28.642 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489992536418697045 TiDB Cluster ID. tikv_gc_leader_uuid 658221c1588000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-c-g3td9, pid:24209, start at 2025-04-06 08:36:28.672989583 +0800 CST m=+2.178983600 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:38:28.680 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:36:28.642 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:26:28.642 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 246 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 4 DDL Table Version. Do not delete. cluster_id 7489992530917601721 TiDB Cluster ID. tikv_gc_leader_uuid 658221c18b0000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-periodics-cdc-integration-mysql-test-234-c-g3td9, pid:24282, start at 2025-04-06 08:36:28.861531566 +0800 CST m=+2.295533325 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20250406-08:38:28.869 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20250406-08:36:28.844 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20250406-08:26:28.844 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v9.0.0-beta.1.pre-129-g8ecb79fcf3 Edition: Community Git Commit Hash: 8ecb79fcf30098e6786f4d0440ead2a879a3ad4a Git Branch: HEAD UTC Build Time: 2025-04-03 03:49:11 Enable Features: jemalloc sm4(GmSSL) mem-profiling avx2 avx512 unwind thinlto hnsw.l2=skylake hnsw.cosine=skylake vec.l2=skylake vec.cos=skylake Profile: RELWITHDEBINFO Compiler: clang++ 17.0.6 Raft Proxy Git Commit Hash: 11578499e429348400a384f1ca08a0200de99f0d Git Commit Branch: HEAD UTC Build Time: "" Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/synced_status/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/synced_status/tiflash/log/error.log arg matches is ArgMatches { args: {"log-file": MatchedArg { occurs: 1, indices: [20], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [22], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash/db/proxy"] }, "engine-version": MatchedArg { occurs: 1, indices: [14], vals: ["v9.0.0-beta.1.pre-129-g8ecb79fcf3"] }, "memory-limit-ratio": MatchedArg { occurs: 1, indices: [6], vals: ["0.800000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [12], vals: ["8ecb79fcf30098e6786f4d0440ead2a879a3ad4a"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [10], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [18], vals: ["127.0.0.1:2379"] }, "engine-label": MatchedArg { occurs: 1, indices: [16], vals: ["tiflash"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + cd /tmp/tidb_cdc_test/synced_status + export 'GO_FAILPOINTS=github.com/pingcap/tiflow/cdc/owner/ChangefeedOwnerNotUpdateCheckpoint=return(true)' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ChangefeedOwnerNotUpdateCheckpoint=return(true)' ++ run_cdc_cli_tso_query 127.0.0.1 2379 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.25544.out cli tso query --pd=http://127.0.0.1:2379 + set +x + tso='457152867386523650 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 457152867386523650 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + start_ts=457152867386523650 + run_cdc_server --workdir /tmp/tidb_cdc_test/synced_status --binary cdc.test [Sun Apr 6 08:36:34 CST 2025] <<<<<< START cdc server in synced_status case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ChangefeedOwnerNotUpdateCheckpoint=return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.2558025582.out server --log-file /tmp/tidb_cdc_test/synced_status/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/synced_status/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 06 Apr 2025 00:36:37 GMT < Content-Length: 820 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5f4d8827-3a7b-420b-ba69-ab3d716769a9 {"id":"5f4d8827-3a7b-420b-ba69-ab3d716769a9","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899795} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/2231960886e523e6 5f4d8827-3a7b-420b-ba69-ab3d716769a9 /tidb/cdc/default/default/upstream/7489992536418697045 {"id":7489992536418697045,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5f4d8827-3a7b-420b-ba69-ab3d716769a9 {"id":"5f4d8827-3a7b-420b-ba69-ab3d716769a9","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899795} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/2231960886e523e6 5f4d8827-3a7b-420b-ba69-ab3d716769a9 /tidb/cdc/default/default/upstream/7489992536418697045 {"id":7489992536418697045,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5f4d8827-3a7b-420b-ba69-ab3d716769a9 {"id":"5f4d8827-3a7b-420b-ba69-ab3d716769a9","address":"127.0.0.1:8300","version":"v9.0.0-beta.1-9-g4c631d595","git-hash":"4c631d5951854ac91765b729fe3849d8e8d5ed8a","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/bin/cdc.test","start-timestamp":1743899795} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/2231960886e523e6 5f4d8827-3a7b-420b-ba69-ab3d716769a9 /tidb/cdc/default/default/upstream/7489992536418697045 {"id":7489992536418697045,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + config_path=conf/changefeed.toml + SINK_URI='mysql://root@127.0.0.1:3306/?max-txn-row=1' + run_cdc_cli changefeed create --start-ts=457152867386523650 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/synced_status/conf/changefeed.toml + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.25642.out cli changefeed create --start-ts=457152867386523650 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/periodics_cdc_integration_mysql_test/tiflow/tests/integration_tests/synced_status/conf/changefeed.toml Create changefeed successfully! ID: test-1 Info: {"upstream_id":7489992536418697045,"namespace":"default","id":"test-1","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2025-04-06T08:36:38.189512991+08:00","start_ts":457152867386523650,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"send-all-bootstrap-at-start":false,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":120,"checkpoint_interval":20}},"state":"normal","creator_version":"v9.0.0-beta.1-9-g4c631d595","resolved_ts":457152867386523650,"checkpoint_ts":457152867386523650,"checkpoint_time":"2025-04-06 08:36:33.192"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x + sleep 20 ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 723 100 723 0 0 10087 0 --:--:-- --:--:-- --:--:-- 10183 + synced_status='{"synced":false,"sink_checkpoint_ts":"2025-04-06 08:36:33.192","puller_resolved_ts":"1970-01-01 08:00:00.000","last_synced_ts":"1970-01-01 08:00:00.000","now_ts":"2025-04-06 08:36:59.000","info":"Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' \u003e '\''Resolved-Ts'\'' \u003e '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait"}' ++ echo '{"synced":false,"sink_checkpoint_ts":"2025-04-06' '08:36:33.192","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2025-04-06' '08:36:59.000","info":"Please' check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view ''\''TiKV-Details'\''' '\u003e' ''\''Resolved-Ts'\''' '\u003e' ''\''Max' Leader Resolved TS 'gap'\''' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please 'wait"}' ++ jq .synced + status=false + '[' false '!=' false ']' ++ echo '{"synced":false,"sink_checkpoint_ts":"2025-04-06' '08:36:33.192","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2025-04-06' '08:36:59.000","info":"Please' check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view ''\''TiKV-Details'\''' '\u003e' ''\''Resolved-Ts'\''' '\u003e' ''\''Max' Leader Resolved TS 'gap'\''' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please 'wait"}' ++ jq -r .info + info='Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' + target_message='Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' + '[' 'Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' '!=' 'Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' ']' + export GO_FAILPOINTS= + GO_FAILPOINTS= + cleanup_process cdc.test wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit + stop_tidb_cluster + check_logs /tmp/tidb_cdc_test/synced_status ++ date + echo '[Sun Apr 6 08:37:12 CST 2025] <<<<<< run test case synced_status success! >>>>>>' [Sun Apr 6 08:37:12 CST 2025] <<<<<< run test case synced_status success! >>>>>> + stop_tidb_cluster \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } [Pipeline] // timeout [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-periodics_cdc_integration_mysql_test-234/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // parallel [Pipeline] } [Pipeline] // stage [Pipeline] stage [Pipeline] { (Declarative: Post Actions) [Pipeline] script [Pipeline] { [Pipeline] withCredentials Masking supported pattern matches of $WEBHOOK_URL [Pipeline] { [Pipeline] sh Warning: A secret was passed to "sh" using Groovy String interpolation, which is insecure. Affected argument(s) used the following variable(s): [WEBHOOK_URL] See https://jenkins.io/redirect/groovy-string-interpolation for details. + curl -X POST **** -H 'Content-Type: application/json' -d '{ "msg_type": "text", "content": { "text": "CDC MySQL Integration Test: SUCCESS\nBranch: master\nBuild URL: https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/periodics_cdc_integration_mysql_test/234/\nJob Page: https://prow.tidb.net/?repo=pingcap%2Ftiflow&type=periodic\n" } }' % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 499 100 77 100 422 62 342 0:00:01 0:00:01 --:--:-- 343 {"StatusCode":0,"StatusMessage":"success","code":0,"data":{},"msg":"success"} [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // script [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] End of Pipeline Finished: SUCCESS